From cee2e40c3389b5261a8c12a2867a1d4d85a73a2b Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Tue, 28 Jul 2026 15:26:36 +0800 Subject: [PATCH 01/37] [CI] Skip known-failing tests and filter NPU ready TTS by npu mark (#5489) --- .buildkite/npu/test-npu-ready.yml | 2 +- .../core/sched/test_omni_ar_scheduler_logprobs.py | 1 + .../test_bagel_shared_memory_connector.py | 1 + .../online_serving/test_qwen3_tts_customvoice.py | 15 +++++++++++---- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.buildkite/npu/test-npu-ready.yml b/.buildkite/npu/test-npu-ready.yml index 956e8acd6..c427ff065 100644 --- a/.buildkite/npu/test-npu-ready.yml +++ b/.buildkite/npu/test-npu-ready.yml @@ -31,4 +31,4 @@ steps: VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" HF_TOKEN: "${HF_TOKEN}" commands: - - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model' + - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model and npu' --run-level 'core_model' diff --git a/tests/core/sched/test_omni_ar_scheduler_logprobs.py b/tests/core/sched/test_omni_ar_scheduler_logprobs.py index 155bd894f..130ac1c88 100644 --- a/tests/core/sched/test_omni_ar_scheduler_logprobs.py +++ b/tests/core/sched/test_omni_ar_scheduler_logprobs.py @@ -104,6 +104,7 @@ class _RequestQueue(list): self.remove(request) +@pytest.mark.skip(reason="#issue 5484") def test_invalid_logprobs_finish_only_the_affected_scheduler_request() -> None: """A bad runner response must not bring down unrelated batch requests.""" bad = _Request("bad") diff --git a/tests/distributed/omni_connectors/test_bagel_shared_memory_connector.py b/tests/distributed/omni_connectors/test_bagel_shared_memory_connector.py index c21af394a..995835024 100644 --- a/tests/distributed/omni_connectors/test_bagel_shared_memory_connector.py +++ b/tests/distributed/omni_connectors/test_bagel_shared_memory_connector.py @@ -228,6 +228,7 @@ def _resolve_deploy_config(config_path: str, run_level: str) -> str: @pytest.mark.advanced_model +@pytest.mark.skip(reason="#issue 5475") @pytest.mark.diffusion @hardware_test(res={"cuda": "H100", "rocm": "MI325"}) def test_bagel_img2img_shared_memory_connector(run_level): diff --git a/tests/e2e/online_serving/test_qwen3_tts_customvoice.py b/tests/e2e/online_serving/test_qwen3_tts_customvoice.py index 62528c272..7ebe08e1a 100644 --- a/tests/e2e/online_serving/test_qwen3_tts_customvoice.py +++ b/tests/e2e/online_serving/test_qwen3_tts_customvoice.py @@ -12,7 +12,6 @@ import os os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" import pytest -from vllm.platforms import current_platform from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniServerParams @@ -21,6 +20,7 @@ from tests.helpers.stage_config import ( get_deploy_config_stage, modify_stage_config, ) +from vllm_omni.platforms import current_omni_platform MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice" @@ -71,7 +71,6 @@ default_tts_server_params = [ @pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.tts -@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA Graph startup test requires CUDA") @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", default_tts_server_params, indirect=True) def test_default_cuda_graph_startup(omni_server) -> None: @@ -93,7 +92,11 @@ def test_default_cuda_graph_startup(omni_server) -> None: @pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.tts -@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.skipif( + current_omni_platform.is_npu(), + reason="#issue 5479", +) +@hardware_test(res={"cuda": "L4", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) def test_text_to_audio_001(omni_server, openai_client) -> None: """ @@ -119,7 +122,11 @@ def test_text_to_audio_001(omni_server, openai_client) -> None: @pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.tts -@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.skipif( + current_omni_platform.is_npu(), + reason="#issue 5479", +) +@hardware_test(res={"cuda": "L4", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) def test_text_to_audio_002(omni_server, openai_client) -> None: """ From 66a9b0c84fa4e56ab0e6d3c7925dd6a68b0ae82d Mon Sep 17 00:00:00 2001 From: Zhou Taichang Date: Tue, 28 Jul 2026 17:14:26 +0800 Subject: [PATCH 02/37] [Bugfix] Select the AR-Diffusion engine in the DreamZero CFG-parallel deploy config (#5439) Co-authored-by: Claude Opus 5 (1M context) --- tests/dreamzero/test_deploy_configs.py | 57 ++++++++++++++++++++++++ vllm_omni/deploy/dreamzero_tp1_cfg2.yaml | 2 + 2 files changed, 59 insertions(+) create mode 100644 tests/dreamzero/test_deploy_configs.py diff --git a/tests/dreamzero/test_deploy_configs.py b/tests/dreamzero/test_deploy_configs.py new file mode 100644 index 000000000..8f9770add --- /dev/null +++ b/tests/dreamzero/test_deploy_configs.py @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Every shipped DreamZero deploy config must select the AR-Diffusion engine. + +``DreamZeroPipeline.__init__`` rejects any other engine, so a config missing +``engine_backend`` fails at worker startup rather than at request time. That +only surfaces in slow multi-GPU e2e serving tests (see issue #5046), so assert +it here on CPU instead. +""" + +from pathlib import Path +from types import SimpleNamespace + +import pytest +import yaml + +from tests.helpers.stage_config import get_deploy_config_path +from vllm_omni.diffusion.diffusion_engine import DiffusionEngine +from vllm_omni.experimental.ar_diffusion.engine import ARDiffusionEngine + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +# Discover siblings of a known config so a future dreamzero*.yaml is covered +# without editing this test. +DEPLOY_DIR = Path(get_deploy_config_path("dreamzero.yaml")).parent + + +def _dreamzero_stages() -> list[tuple[str, int, dict]]: + stages = [] + for name in sorted(path.name for path in DEPLOY_DIR.glob("dreamzero*.yaml")): + config = yaml.safe_load(Path(get_deploy_config_path(name)).read_text()) + for index, stage in enumerate(config.get("stages") or []): + stages.append((name, index, stage)) + return stages + + +def test_dreamzero_deploy_configs_exist() -> None: + assert _dreamzero_stages(), f"no DreamZero deploy configs found under {DEPLOY_DIR}" + + +@pytest.mark.parametrize( + ("config_name", "stage_index", "stage"), + _dreamzero_stages(), + ids=lambda value: value if isinstance(value, str) else None, +) +def test_dreamzero_stage_selects_ar_diffusion_engine(config_name: str, stage_index: int, stage: dict) -> None: + backend = stage.get("engine_backend") + assert backend, ( + f"{config_name} stage {stage_index} does not set engine_backend; " + "DreamZeroPipeline requires the AR-Diffusion engine" + ) + # Resolving also proves the import path is real, not just a lookalike string. + engine_class = DiffusionEngine.resolve_engine_class(SimpleNamespace(engine_backend=backend)) + assert issubclass(engine_class, ARDiffusionEngine), ( + f"{config_name} stage {stage_index} resolves to {engine_class.__name__}, not an AR-Diffusion engine" + ) diff --git a/vllm_omni/deploy/dreamzero_tp1_cfg2.yaml b/vllm_omni/deploy/dreamzero_tp1_cfg2.yaml index a453d43f3..0bf3d9517 100644 --- a/vllm_omni/deploy/dreamzero_tp1_cfg2.yaml +++ b/vllm_omni/deploy/dreamzero_tp1_cfg2.yaml @@ -11,6 +11,8 @@ stages: max_num_seqs: 1 enforce_eager: false model_class_name: DreamZeroPipeline + # DreamZero runs on the AR-Diffusion engine (engine-level paged KV cache). + engine_backend: vllm_omni.experimental.ar_diffusion.engine.ARDiffusionEngine parallel_config: tensor_parallel_size: 1 cfg_parallel_size: 2 From 12d75cf63a026dc20eacbd42fede9734da460a36 Mon Sep 17 00:00:00 2001 From: Mu GuanLin <1203789601@qq.com> Date: Tue, 28 Jul 2026 18:44:25 +0800 Subject: [PATCH 03/37] [Feature] Add official LTX multimodal guidance and precision guard (#5148) Signed-off-by: mglyn <1203789601@qq.com> --- .buildkite/cuda/test-nightly.yml | 2 +- docs/models/supported_models.md | 8 +- docs/serving/videos_api.md | 2 +- .../offline_inference/image_to_video.md | 26 +- .../offline_inference/text_to_video.md | 46 +- .../examples/online_serving/text_to_video.md | 98 +- .../image_to_video/README.md | 53 +- .../image_to_video/image_to_video.py | 47 +- .../text_to_video/text_to_video.md | 46 +- .../text_to_video/text_to_video.py | 44 +- .../online_serving/text_to_video/README.md | 76 +- .../text_to_video/run_curl_ltx2.sh | 1 - .../text_to_video/run_server_ltx2.sh | 8 - recipes/LTX/LTX-2.3.md | 154 --- recipes/LTX/LTX-2.md | 408 ++++-- recipes/README.md | 3 +- .../dmd2/test_dmd2_request_sanitization.py | 3 +- .../models/dmd2/test_dmd2_scheduler.py | 3 +- .../models/ltx2/test_ltx2_conditioning.py | 244 +++- .../models/ltx2/test_ltx2_latents.py | 58 +- .../models/ltx2/test_ltx2_parallel.py | 143 +- .../models/ltx2/test_ltx2_pipeline.py | 1204 ++++++++++++++--- .../models/ltx2/test_ltx2_transformer.py | 18 +- tests/diffusion/models/ltx2/test_ltx2_vae.py | 40 +- tests/e2e/accuracy/ltx/__init__.py | 1 + tests/e2e/accuracy/ltx/run_ltx_reference.py | 331 +++++ .../ltx/test_ltx_official_similarity.py | 440 ++++++ .../accuracy/test_ltx2_3_video_similarity.py | 413 ------ .../e2e/online_serving/test_ltx2_expansion.py | 28 +- .../test_ltx2_cfg_parallel_parity.py | 243 ---- tests/model_extras/test_model_extras.py | 30 + tests/model_tests/diffusion/test_alignment.py | 6 +- .../model_loader/diffusers_loader.py | 2 +- vllm_omni/diffusion/models/ltx2/__init__.py | 16 +- .../diffusion/models/ltx2/ltx2_components.py | 209 ++- .../models/ltx2/ltx2_conditioning.py | 461 +++++-- .../diffusion/models/ltx2/ltx2_denoise.py | 177 ++- .../diffusion/models/ltx2/ltx2_guidance.py | 673 ++++----- .../diffusion/models/ltx2/ltx2_latents.py | 64 +- .../diffusion/models/ltx2/ltx2_recipes.py | 165 ++- .../diffusion/models/ltx2/ltx2_request.py | 321 ++++- ...x2_pipeline_runtime.py => ltx2_runtime.py} | 370 ++++- .../diffusion/models/ltx2/ltx2_transformer.py | 145 +- .../diffusion/models/ltx2/pipeline_ltx2.py | 238 +--- .../ltx2/pipeline_ltx2_latent_upsample.py | 262 ---- .../models/ltx2/pipeline_ltx2_two_stage.py | 204 +-- vllm_omni/diffusion/registry.py | 32 +- vllm_omni/diffusion/worker/request_batch.py | 3 +- vllm_omni/entrypoints/cli/serve.py | 2 +- vllm_omni/model_extras/ltx2.py | 27 + vllm_omni/model_extras/registry.py | 11 + 51 files changed, 4547 insertions(+), 3062 deletions(-) delete mode 100644 recipes/LTX/LTX-2.3.md create mode 100644 tests/e2e/accuracy/ltx/__init__.py create mode 100644 tests/e2e/accuracy/ltx/run_ltx_reference.py create mode 100644 tests/e2e/accuracy/ltx/test_ltx_official_similarity.py delete mode 100644 tests/e2e/accuracy/test_ltx2_3_video_similarity.py delete mode 100644 tests/examples/offline_inference/test_ltx2_cfg_parallel_parity.py rename vllm_omni/diffusion/models/ltx2/{ltx2_pipeline_runtime.py => ltx2_runtime.py} (59%) delete mode 100644 vllm_omni/diffusion/models/ltx2/pipeline_ltx2_latent_upsample.py create mode 100644 vllm_omni/model_extras/ltx2.py diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index 2aed12819..a82a4aab5 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -426,7 +426,7 @@ steps: commands: - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m full_model --run-level full_model - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model - - pytest -s -v tests/e2e/accuracy/test_ltx2_3_video_similarity.py -m full_model --run-level full_model + - pytest -s -v 'tests/e2e/accuracy/ltx/test_ltx_official_similarity.py::test_ltx_one_stage_matches_official[ltx2_3_i2v]' -m full_model --run-level full_model - pytest -s -v tests/e2e/accuracy/test_video_streaming_output_similarity.py -m full_model --run-level full_model mirror_hardwares: h100_2 diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 9f99592fd..021df65ce 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -41,12 +41,8 @@ th { | `Wan22S2VPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | | `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | | `Wan22VACEPipeline` | Wan2.2-VACE | `Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers` | ✅︎ | | | | -| `LTX2Pipeline` | LTX-2-T2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | | -| `LTX2ImageToVideoPipeline` | LTX-2-I2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | | -| `LTX2TwoStagesPipeline` | LTX-2-T2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | | -| `LTX2ImageToVideoTwoStagesPipeline` | LTX-2-I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | | -| `LTX23Pipeline` | LTX-2.3-T2V | `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | | -| `LTX23ImageToVideoPipeline` | LTX-2.3-I2V | `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | | +| `LTX2Pipeline` | LTX-2 / LTX-2.3 one-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | | +| `LTX2DistilledPipeline` | LTX-2 distilled two-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | | | `LingBotVideoPipeline` | LingBot-Video dense and MoE T2V | `robbyant/lingbot-video-dense-1.3b`, `robbyant/lingbot-video-moe-30b-a3b` | ✅︎ | | | | | `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | | `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | | diff --git a/docs/serving/videos_api.md b/docs/serving/videos_api.md index 72d79c304..6a94f9dab 100644 --- a/docs/serving/videos_api.md +++ b/docs/serving/videos_api.md @@ -73,7 +73,7 @@ curl -L "http://localhost:8091/v1/videos/${video_id}/content" -o output.mp4 | `audio_reference` | string | null | JSON-encoded audio reference for speech-to-video: `{"audio_url": "..."}` — supports HTTP(s) URLs or base64 data URLs | | `width` | integer | model default | Output video width | | `height` | integer | model default | Output video height | -| `num_frames` | integer | model default | Number of generated frames | +| `num_frames` | integer | 1 | Number of generated frames | | `fps` | integer | model default | Output frames per second | | `num_inference_steps` | integer | model default | Number of diffusion steps | | `guidance_scale` | number | null | CFG guidance scale for the low-noise stage | diff --git a/docs/user_guide/examples/offline_inference/image_to_video.md b/docs/user_guide/examples/offline_inference/image_to_video.md index b3d76ac46..db7a31ce5 100644 --- a/docs/user_guide/examples/offline_inference/image_to_video.md +++ b/docs/user_guide/examples/offline_inference/image_to_video.md @@ -4,7 +4,7 @@ Source argparse.Namespace: parser.add_argument( "--model-class-name", default=None, - help="Override model class name (e.g., LTX2ImageToVideoPipeline or LTX23ImageToVideoPipeline).", + help="Override model class name (LTX checkpoints default to LTX2Pipeline).", ) parser.add_argument( "--deploy-config", @@ -117,7 +115,7 @@ def parse_args() -> argparse.Namespace: help="Path to a reference image. Repeat to provide multiple references.", ) parser.add_argument("--prompt", default="", help="Text prompt describing the desired motion.") - parser.add_argument("--negative-prompt", default="", help="Negative prompt.") + parser.add_argument("--negative-prompt", default=None, help="Negative prompt. Default: model-specific.") parser.add_argument("--seed", type=int, default=42, help="Random seed.") parser.add_argument("--guidance-scale", type=float, default=None, help="CFG scale. Default: model-specific.") parser.add_argument( @@ -331,7 +329,10 @@ def main(): generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed) model_name = str(args.model).lower() if args.model is not None else "" model_class_name = args.model_class_name - is_ltx2 = model_class_name in {"LTX2ImageToVideoPipeline", "LTX23ImageToVideoPipeline"} + model_class_name_lower = (model_class_name or "").lower() + is_ltx2_distilled = "distilled" in model_class_name_lower or "distilled" in model_name + is_ltx23 = "ltx23" in model_class_name_lower or "ltx-2.3" in model_name + is_ltx2 = is_ltx2_distilled or is_ltx23 or "ltx2" in model_class_name_lower or "ltx-2" in model_name is_cosmos = "cosmos" in model_name or (model_class_name is not None and "cosmos" in model_class_name.lower()) image = PIL.Image.open(args.image).convert("RGB") if args.image else None @@ -356,13 +357,23 @@ def main(): 1280 * 720, 16, ) + elif is_ltx2_distilled: + d_fps, d_guidance, d_num_frames, d_steps, d_flow_shift, d_max_area, d_mod = ( + 24, + None, + 121, + 8, + None, + 1024 * 1536, + 64, + ) elif is_ltx2: d_fps, d_guidance, d_num_frames, d_steps, d_flow_shift, d_max_area, d_mod = ( 24, - 4.0, + None, 121, - 40, - 5.0, + 30 if is_ltx23 else 40, + None, 512 * 768, 32, ) @@ -445,7 +456,6 @@ def main(): vae_use_slicing=args.vae_use_slicing, vae_use_tiling=args.vae_use_tiling, boundary_ratio=args.boundary_ratio, - flow_shift=flow_shift, diffusion_kv_cache_dtype=args.diffusion_kv_cache_dtype, diffusion_kv_cache_skip_steps=args.diffusion_kv_cache_skip_steps, diffusion_kv_cache_skip_layers=args.diffusion_kv_cache_skip_layers, @@ -460,6 +470,8 @@ def main(): ) if args.deploy_config: omni_kwargs["deploy_config"] = args.deploy_config + if flow_shift is not None: + omni_kwargs["flow_shift"] = flow_shift if args.quantization is not None: omni_kwargs["quantization"] = args.quantization # Cosmos3 loads its (gated) guardrail models at build time, so the guardrails @@ -492,10 +504,14 @@ def main(): print(f" Video size: {width}x{height}") print(f"{'=' * 60}\n") + negative_prompt = args.negative_prompt + if negative_prompt is None and not is_ltx2: + # Preserve the historical empty-prompt behavior for non-LTX examples. + negative_prompt = "" prompt_dict = build_image_to_video_prompt( model_class_name=model_class_name, prompt=args.prompt, - negative_prompt=args.negative_prompt, + negative_prompt=negative_prompt, media_inputs=media_inputs, height=height, width=width, @@ -511,11 +527,10 @@ def main(): num_inference_steps=num_inference_steps, num_frames=num_frames, frame_rate=frame_rate, - extra_args={ - "sample_solver": args.sample_solver, - "flow_shift": flow_shift, - }, + extra_args={"sample_solver": args.sample_solver}, ) + if flow_shift is not None: + sampling_params.extra_args["flow_shift"] = flow_shift # Route model-specific knobs through extra_body, filtered against the model's # declared extra_body_params. Models without a declaration only forward explicit diff --git a/examples/offline_inference/text_to_video/text_to_video.md b/examples/offline_inference/text_to_video/text_to_video.md index 876b8429b..0df1f9bd0 100644 --- a/examples/offline_inference/text_to_video/text_to_video.md +++ b/examples/offline_inference/text_to_video/text_to_video.md @@ -8,7 +8,7 @@ A unified script for text-to-video generation. Supports multiple models with mod |---|---|---|---|---|---| | `Wan-AI/Wan2.1-VACE-1.3B-diffusers` | 480x832 | 81 | 30 | 5.0 | ~20 GiB (RTX 5090, VAE tiling) | | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | 720x1280 | 81 | 40 | 4.0 | ~60 GiB | -| `diffusers/LTX-2.3-Diffusers` | 384x512 | 25 | 20 | 4.0 | 96GB-class GPU | +| `Lightricks/LTX-2` | 512x768 | 121 | 40 | video 3.0 / audio 7.0 | Model-dependent | | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v` | 480x832 | 121 | 50 | 6.0 | 1×A100 80GB | | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | 720x1280 | 121 | 50 | 6.0 | FP8 + VAE tiling required | | `nvidia/Cosmos3-Nano` | 720x1280 | 189 | 35 | 6.0 | ~46 GiB (peak, 720p) | @@ -55,45 +55,17 @@ python text_to_video.py \ --output vace_t2v_output.mp4 ``` -LTX2 example: +### LTX-2 ```bash python text_to_video.py \ - --model "Lightricks/LTX-2" \ - --prompt "A cinematic close-up of ocean waves at golden hour." \ - --negative-prompt "worst quality, inconsistent motion, blurry, jittery, distorted" \ - --height 512 \ - --width 768 \ - --num-frames 121 \ - --num-inference-steps 40 \ - --guidance-scale 4.0 \ - --frame-rate 24 \ - --output ltx2_out.mp4 -``` - -### LTX-2.3 - -```bash -python text_to_video.py \ - --model diffusers/LTX-2.3-Diffusers \ - --model-class-name LTX23Pipeline \ + --model Lightricks/LTX-2 \ --prompt "Cherry blossoms swaying gently in the breeze with synchronized ambient sound" \ - --negative-prompt "worst quality, inconsistent motion, blurry, jittery, distorted" \ - --height 384 \ - --width 512 \ - --num-frames 25 \ - --num-inference-steps 20 \ - --guidance-scale 4.0 \ - --frame-rate 24 \ - --fps 24 \ - --audio-sample-rate 48000 \ - --output ltx23_t2v_output.mp4 + --output ltx2_output.mp4 ``` -Use the Diffusers-format checkpoint `diffusers/LTX-2.3-Diffusers`; the -upstream `Lightricks/LTX-2.3` raw safetensors repo is not directly loadable by -this pipeline. Pass `--model-class-name LTX23Pipeline` to select the LTX-2.3 -text-to-video pipeline explicitly. +See the [LTX-2 recipe](../../../recipes/LTX/LTX-2.md) for all checkpoints, +pipeline selection, I2V, defaults, and advanced options. ### HunyuanVideo-1.5 (480p) @@ -212,8 +184,7 @@ python text_to_video.py \ ### Common - `--model`: Diffusers model ID or local path. -- `--model-class-name`: Optional explicit pipeline class. Use `LTX23Pipeline` - for LTX-2.3 text-to-video. +- `--model-class-name`: Optional explicit pipeline override. - `--prompt`: text description (string). - `--height/--width`: output resolution. Default depends on model. - `--num-frames`: number of frames. Default depends on model. @@ -229,8 +200,7 @@ python text_to_video.py \ - `--enable-cpu-offload`: enable CPU offloading for diffusion models. - `--enable-layerwise-offload`: enable layerwise offloading on DiT modules. - `--frame-rate`: generation FPS for pipelines that require it (e.g., LTX2). -- `--audio-sample-rate`: audio sample rate for embedded audio (when the - pipeline returns audio; LTX-2.3 outputs 48kHz audio). +- `--audio-sample-rate`: fallback audio sample rate when the pipeline returns audio. - `--quantization`: quantization method (such as `fp8` for FP8). - `--flow-shift`: scheduler flow_shift parameter. - `--extra-body`: JSON object of model-specific generation params, filtered against the model's declared `extra_body_params` (see [`vllm_omni/model_extras`](../../../vllm_omni/model_extras)). Used by Cosmos3 (see above). diff --git a/examples/offline_inference/text_to_video/text_to_video.py b/examples/offline_inference/text_to_video/text_to_video.py index addf7cb7a..b0c8a2509 100644 --- a/examples/offline_inference/text_to_video/text_to_video.py +++ b/examples/offline_inference/text_to_video/text_to_video.py @@ -65,11 +65,42 @@ _MODEL_PRESETS = { "fps": 16, "output": "helios_output.mp4", }, + "ltx2": { + "height": 512, + "width": 768, + "num_frames": 121, + "num_inference_steps": 40, + "fps": 24, + "output": "ltx2_output.mp4", + }, + "ltx2_distilled": { + "height": 1024, + "width": 1536, + "num_frames": 121, + "num_inference_steps": 8, + "fps": 24, + "output": "ltx2_distilled_output.mp4", + }, + "ltx23": { + "height": 512, + "width": 768, + "num_frames": 121, + "num_inference_steps": 30, + "fps": 24, + "output": "ltx23_output.mp4", + }, } -def _detect_preset(model: str) -> dict: +def _detect_preset(model: str, model_class_name: str | None = None) -> dict: model_lower = model.lower() + class_lower = (model_class_name or "").lower() + if "distilled" in class_lower or "distilled" in model_lower: + return _MODEL_PRESETS["ltx2_distilled"] + if "ltx23" in class_lower or "ltx-2.3" in model_lower or "ltx_2.3" in model_lower: + return _MODEL_PRESETS["ltx23"] + if "ltx2" in class_lower or "ltx-2" in model_lower or "ltx_2" in model_lower: + return _MODEL_PRESETS["ltx2"] if "vace" in model_lower: return _MODEL_PRESETS["vace"] if "cosmos" in model_lower: @@ -122,7 +153,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--model-class-name", default=None, - help="Override model class name (e.g., LTX2TwoStagesVideoPipeline).", + help="Override model class name (e.g., LTX2Pipeline).", ) parser.add_argument( "--deploy-config", @@ -130,7 +161,7 @@ def parse_args() -> argparse.Namespace: help="Optional deploy config YAML to use for pipeline-backed runs.", ) parser.add_argument("--prompt", default="A serene lakeside sunrise with mist over the water.", help="Text prompt.") - parser.add_argument("--negative-prompt", default="", help="Negative prompt.") + parser.add_argument("--negative-prompt", default=None, help="Negative prompt. Default: model-specific.") parser.add_argument( "--extra-body", type=parse_extra_body, @@ -331,7 +362,7 @@ def main(): args = parse_args() model_class_name = args.model_class_name - preset = _detect_preset(args.model) + preset = _detect_preset(args.model, model_class_name) for key, default_val in preset.items(): if getattr(args, key.replace("-", "_"), None) is None: setattr(args, key.replace("-", "_"), default_val) @@ -422,8 +453,11 @@ def main(): print(f"{'=' * 60}\n") prompt_dict = {"prompt": args.prompt} - if args.negative_prompt: + if args.negative_prompt is not None: prompt_dict["negative_prompt"] = args.negative_prompt + elif preset not in (_MODEL_PRESETS["ltx2"], _MODEL_PRESETS["ltx23"]): + # Preserve the historical empty-prompt behavior for non-LTX examples. + prompt_dict["negative_prompt"] = "" sampling_kwargs = dict( height=args.height, diff --git a/examples/online_serving/text_to_video/README.md b/examples/online_serving/text_to_video/README.md index 71afd9e7c..51512fe0c 100644 --- a/examples/online_serving/text_to_video/README.md +++ b/examples/online_serving/text_to_video/README.md @@ -248,82 +248,12 @@ done ## LTX-2 -### Start Server - -#### Basic Start - ```bash -vllm serve Lightricks/LTX-2 --omni --port 8098 \ - --enforce-eager --boundary-ratio 1.0 +vllm serve Lightricks/LTX-2 --omni --port 8098 ``` -#### Start with Optimization Presets - -Use the LTX-2 startup script with built-in optimization presets: - -```bash -# Baseline (1 GPU, eager) -bash run_server_ltx2.sh baseline - -# 4-GPU Ulysses sequence parallelism (lossless) -bash run_server_ltx2.sh ulysses4 - -# Cache-DiT lossy acceleration (1 GPU, ~1.4× speedup) -bash run_server_ltx2.sh cache-dit - -# Best combo: 4-GPU Ulysses SP + Cache-DiT (~2.2× speedup) -bash run_server_ltx2.sh best-combo -``` - -#### Optimization Benchmarks - -Benchmarked on H800, online serving (480×768, 41 frames, 20 steps, `seed=42`). -"Inference" is the server-reported inference time; excludes HTTP/poll overhead. - -| Preset | Server Command | Inference (s) | Speedup | Type | -|--------|---------------|---------------|---------|------| -| `baseline` | `--enforce-eager` | 10.3 | 1.00× | — | -| `compile` | *(default, no --enforce-eager)* | ~10.3 (warm) | ~1.00× | Lossless | -| `ulysses4` | `--enforce-eager --usp 4` | ~10.3 | ~1.00× | Lossless | -| `cache-dit` | `--enforce-eager --cache-backend cache_dit` | 7.4 avg | ~1.4× | Lossy | -| `best-combo` | `--enforce-eager --usp 4 --cache-backend cache_dit` | 4.7 avg | **~2.2×** | Lossless + Lossy | - -**Observations**: -- **torch.compile**: On H800, warm-request inference time matches the eager baseline (~10.3s). - The first request pays ~6s compilation overhead. Benefit depends on model architecture and GPU. -- **Ulysses SP (4 GPU)**: No measurable speedup alone for 41-frame generation at this resolution. - Communication overhead outweighs gains at this sequence length. -- **Cache-DiT**: Inference varies per request (6–10s) due to dynamic caching decisions. - Average is ~7.4s (~1.4× speedup) with slight quality tradeoff. -- **Best combo**: 4-GPU Ulysses SP + Cache-DiT synergize well — Cache-DiT reduces per-step - computation, making the communication overhead of Ulysses SP worthwhile. Average ~4.7s - (~2.2× speedup). -- **FP8 quantization**: Reduces VRAM but does not speed up LTX-2 on H800 (compute-bound). - -**Deployment Recommendations**: -- For **production with quality priority**: use `baseline` with `--enforce-eager` -- For **maximum throughput** (4 GPUs, quality tradeoff): use `best-combo` (~2.2× speedup) -- For **single-GPU throughput**: use `cache-dit` (~1.4× speedup) -- `--enforce-eager` is recommended to avoid torch.compile warmup latency on first request - -### Send Requests (curl) - -```bash -# Using the provided script -bash run_curl_ltx2.sh - -# Or directly -curl -sS -X POST http://localhost:8098/v1/videos \ - -H "Accept: application/json" \ - -F "prompt=A serene lakeside sunrise with mist over the water." \ - -F "width=768" \ - -F "height=480" \ - -F "num_frames=41" \ - -F "fps=24" \ - -F "num_inference_steps=20" \ - -F "guidance_scale=3.0" \ - -F "seed=42" -``` +See the [LTX-2 recipe](../../../recipes/LTX/LTX-2.md) for all checkpoints, +pipeline selection, requests, defaults, and advanced options. ## Helios diff --git a/examples/online_serving/text_to_video/run_curl_ltx2.sh b/examples/online_serving/text_to_video/run_curl_ltx2.sh index b82f672ea..1cf828bba 100644 --- a/examples/online_serving/text_to_video/run_curl_ltx2.sh +++ b/examples/online_serving/text_to_video/run_curl_ltx2.sh @@ -22,7 +22,6 @@ create_response=$( -F "num_frames=41" \ -F "fps=24" \ -F "num_inference_steps=20" \ - -F "guidance_scale=3.0" \ -F "seed=42" ) diff --git a/examples/online_serving/text_to_video/run_server_ltx2.sh b/examples/online_serving/text_to_video/run_server_ltx2.sh index f4597d3cd..da9351f7f 100644 --- a/examples/online_serving/text_to_video/run_server_ltx2.sh +++ b/examples/online_serving/text_to_video/run_server_ltx2.sh @@ -21,8 +21,6 @@ set -euo pipefail MODEL="${MODEL:-Lightricks/LTX-2}" PORT="${PORT:-8098}" -FLOW_SHIFT="${FLOW_SHIFT:-1.0}" -BOUNDARY_RATIO="${BOUNDARY_RATIO:-1.0}" PRESET="${1:-baseline}" @@ -66,19 +64,13 @@ case "$PRESET" in echo "Environment variables:" echo " MODEL - Model path (default: Lightricks/LTX-2)" echo " PORT - Server port (default: 8098)" - echo " FLOW_SHIFT - Scheduler flow shift (default: 1.0)" - echo " BOUNDARY_RATIO - Boundary ratio (default: 1.0)" exit 1 ;; esac echo "Model: $MODEL" echo "Port: $PORT" -echo "Flow shift: $FLOW_SHIFT" -echo "Boundary ratio: $BOUNDARY_RATIO" vllm serve "$MODEL" --omni \ --port "$PORT" \ - --flow-shift "$FLOW_SHIFT" \ - --boundary-ratio "$BOUNDARY_RATIO" \ "${EXTRA_ARGS[@]}" diff --git a/recipes/LTX/LTX-2.3.md b/recipes/LTX/LTX-2.3.md deleted file mode 100644 index 471ee7ae0..000000000 --- a/recipes/LTX/LTX-2.3.md +++ /dev/null @@ -1,154 +0,0 @@ -# LTX-2.3 - -> 22B parameter text-to-video + audio generation model served via vLLM-Omni - -## Summary - -- Vendor: Lightricks -- Model: `diffusers/LTX-2.3-Diffusers` -- Task: Text-to-video and image-to-video with synchronized audio generation -- Mode: Online serving (pure diffusion) -- Maintainer: @oglok - -## When to use this recipe - -Use this recipe when you want to serve LTX-2.3 for text-to-video or -image-to-video generation with audio. The model generates videos up to 20+ -seconds at 768x512 resolution with 48kHz audio from text prompts, and can -condition video generation on an initial image. Start with a 96GB-class GPU for -validation because the model combines a 22B parameter transformer with text -encoder, VAE, and vocoder components. - -## References - -- Upstream raw checkpoints: -- Diffusers-format checkpoint: -- Requires `diffusers >= 0.38.0` (install from git: `pip install git+https://github.com/huggingface/diffusers.git`) - -## Serving - -### Command - -```bash -vllm serve diffusers/LTX-2.3-Diffusers \ - --omni \ - --model-class-name LTX23Pipeline \ - --stage-init-timeout 600 -``` - -For image-to-video: - -```bash -vllm serve diffusers/LTX-2.3-Diffusers \ - --omni \ - --model-class-name LTX23ImageToVideoPipeline \ - --stage-init-timeout 600 -``` - -### T2V Verification - -```bash -# Health check -curl http://localhost:8000/health - -# Generate a 3-second video (81 frames at 24fps) -curl -X POST http://localhost:8000/v1/videos \ - -F "prompt=A majestic bald eagle soaring over a misty mountain valley at dawn, golden sunlight breaking through clouds" \ - -F "negative_prompt=blurry, low quality, distorted, watermark" \ - -F "model=diffusers/LTX-2.3-Diffusers" \ - -F "num_frames=81" \ - -F "fps=24" \ - -F "size=768x512" \ - -F "num_inference_steps=30" \ - -F "guidance_scale=4.0" \ - -F "seed=42" - -# Generate a 10-second video (241 frames) -curl -X POST http://localhost:8000/v1/videos \ - -F "prompt=A cozy Japanese ramen shop at night in the rain, steam rising from bowls, neon signs reflecting on wet cobblestone streets" \ - -F "model=diffusers/LTX-2.3-Diffusers" \ - -F "num_frames=241" \ - -F "fps=24" \ - -F "size=768x512" \ - -F "num_inference_steps=30" \ - -F "guidance_scale=4.0" - -# Generate a 20-second video (481 frames) -curl -X POST http://localhost:8000/v1/videos \ - -F "prompt=An underwater coral reef teeming with tropical fish, sea turtles gliding gracefully, National Geographic documentary style" \ - -F "model=diffusers/LTX-2.3-Diffusers" \ - -F "num_frames=481" \ - -F "fps=24" \ - -F "size=768x512" \ - -F "num_inference_steps=30" \ - -F "guidance_scale=4.0" -``` - -### I2V Verification - -Start the server with `--model-class-name LTX23ImageToVideoPipeline`, then pass -an image reference. The I2V pipeline requires `input_reference` or -`image_reference` because the first frame is the conditioning input. - -```bash -curl http://localhost:8000/health - -curl -X POST http://localhost:8000/v1/videos/sync \ - -F "prompt=A plush toy astronaut gently waving while the camera slowly pushes in." \ - -F "input_reference=@/absolute/path/to/reference.png" \ - -F "model=diffusers/LTX-2.3-Diffusers" \ - -F "num_frames=81" \ - -F "fps=24" \ - -F "size=768x512" \ - -F "num_inference_steps=30" \ - -F "guidance_scale=4.0" \ - -F "seed=42" \ - -o ltx23_i2v.mp4 -``` - -### Notes - -- Memory usage: run a latest-head serving smoke or DFX benchmark on the target - GPU before publishing load/peak VRAM numbers. -- Checkpoint format: use a Diffusers-format checkpoint such as - `diffusers/LTX-2.3-Diffusers`; the upstream `Lightricks/LTX-2.3` repository ships - raw safetensors and does not contain the subfolder configs required by this - pipeline loader. -- Key flags: - - `--stage-init-timeout 600`: Gives the initial `torch.compile` warmup enough time on large checkpoints - - `--model-class-name LTX23Pipeline`: Selects the LTX-2.3 text-to-video pipeline (not LTX-2) - - `--model-class-name LTX23ImageToVideoPipeline`: Selects the LTX-2.3 image-to-video pipeline - - `input_reference=@...`: Uploads the I2V reference image directly -- Audio: 48kHz AAC via BWE vocoder, automatically synced with video -- CPU offloading: Text encoder (Gemma-3-12B), connectors, VAE, audio VAE, and vocoder stay on CPU and are moved to GPU only when needed -- Supported resolutions: 768x512, 512x384 (must be divisible by 32) -- Frame rate: 24 fps -- Duration: Controlled by `num_frames` (frames = duration_seconds * 24 + 1) -- Known limitations: - - LTX-2.3 I2V currently supports first-frame image conditioning, matching the LTX image-conditioning contract. - - Requires `diffusers >= 0.38.0` (not yet on PyPI, install from git) - -## Hardware Support - -## GPU - -### 1x NVIDIA RTX PRO 6000 Blackwell (96GB) - -#### Environment - -- OS: Ubuntu 22.04 -- Python: 3.10+ -- Driver / runtime: CUDA 13.0, Driver 580.126.09 -- vLLM version: Match the PR or release checkout being validated -- vLLM-Omni version or commit: Use the commit you are deploying from - -### Benchmarking - -Do not copy latency or VRAM numbers between commits or machines. Record them -only after running a latest-head sweep on the target hardware. - -For formal PR benchmarking, reuse -`tests/dfx/perf/tests/test_ltx2_vllm_omni.json` with -`tests/dfx/perf/scripts/run_diffusion_benchmark.py`. Each case in that file includes a JSON **`mark`** (`hardware_marks` + `full_model` + `diffusion`) for local `-m` filtering. That config captures the -single-device eager baseline and CFG-parallel=2 case for a small 384x512, -25-frame, 20-step workload. diff --git a/recipes/LTX/LTX-2.md b/recipes/LTX/LTX-2.md index df049f5c6..ca3201dc1 100644 --- a/recipes/LTX/LTX-2.md +++ b/recipes/LTX/LTX-2.md @@ -1,121 +1,303 @@ -# LTX-2 +# LTX-2 Family -> Text-to-video and image-to-video serving +> LTX-2 and LTX-2.3 text-to-video and image-to-video generation with synchronized audio -## Summary +## Pipelines -- Vendor: Lightricks -- Model: `Lightricks/LTX-2` -- Task: Text-to-video generation and image-to-video generation -- Mode: Online serving with the OpenAI-compatible video API -- Maintainer: Community +| Model | Checkpoint | Task | `--model-class-name` | Batching | +|---|---|---|---|---| +| LTX-2 | `Lightricks/LTX-2` | One-stage T2V/I2V | `LTX2Pipeline` | Yes | +| LTX-2 distilled | `rootonchair/LTX-2-19b-distilled` | Two-stage T2V/I2V | `LTX2DistilledPipeline` | No | +| LTX-2.3 | `diffusers/LTX-2.3-Diffusers` | One-stage T2V/I2V | `LTX2Pipeline` | Yes | -## When to use this recipe +`LTX2Pipeline` is the unified one-stage entry. Checkpoint metadata selects the +LTX-2 or LTX-2.3 profile; omitting an image selects T2V, while one initial +image selects I2V. Both one-stage repositories declare this class, so +`--model-class-name` is optional. LTX-2.3 requires the Diffusers checkpoint; +the raw `Lightricks/LTX-2.3` safetensors repository is not directly loadable. -Use this recipe when you want to deploy the base `Lightricks/LTX-2` model -for both text-to-video and image-to-video generation. The same model ID is used for -both tasks; switch the pipeline class with `--model-class-name` depending on -whether you want T2V or I2V. +`LTX2DistilledPipeline` is the unified distilled entry: omit `image` for T2V +or provide one initial image for I2V. As in the official LTX pipelines, +distilled mode is selected explicitly rather than inferred from the checkpoint +name or metadata; always pass `--model-class-name LTX2DistilledPipeline`. + +## API Migration + +Only `req` may be passed positionally to `LTX2Pipeline`; every optional +`forward` argument is keyword-only: + +```python +# No longer supported +pipe(req, prompt) +pipe(req, image, prompt) + +# Supported +pipe(req, prompt=prompt) +pipe(req, image=image, prompt=prompt) +``` + +The consolidation also removes these registry names without aliases: + +| Removed name | Replacement | +|---|---| +| `LTX23Pipeline` | `LTX2Pipeline`; checkpoint metadata selects LTX-2.3 | +| `LTX2ImageToVideoPipeline` | `LTX2Pipeline` with `image=` | +| `LTX23ImageToVideoPipeline` | `LTX2Pipeline` with `image=`; checkpoint metadata selects LTX-2.3 | +| `LTX2TwoStagesPipeline` | `LTX2DistilledPipeline` | +| `LTX2ImageToVideoTwoStagesPipeline` | `LTX2DistilledPipeline` with `image=` | + +Passing any second positional argument now raises `TypeError`. These changes +affect direct Python callers and explicit `--model-class-name` overrides; +offline and serving entrypoints already use named fields and are unaffected. + +## One-Stage Defaults + +| Parameter | LTX-2 | LTX-2.3 | +|---|---:|---:| +| Width × height | 768 × 512 | 768 × 512 | +| Frames / frame rate | 121 / 24 | 121 / 24 | +| Denoise steps | 40 | 30 | +| Video/audio CFG | 3.0 / 7.0 | 3.0 / 7.0 | +| Video/audio STG | 1.0 / 1.0 | 1.0 / 1.0 | +| Video/audio modality guidance | 3.0 / 3.0 | 3.0 / 3.0 | +| Video/audio rescale | 0.7 / 0.7 | 0.7 / 0.7 | +| Video/audio STG blocks | `[29]` / `[29]` | `[28]` / `[28]` | + +The model recipe also supplies the default negative prompt. Top-level +`guidance_scale` overrides both video and audio CFG values. The online video +API defaults `num_frames` to `1`, so set it explicitly; the offline LTX scripts +default to `121`. + +## Distilled Two-Stage Defaults + +| Parameter | Value | +|---|---:| +| Final width × height | 1536 × 1024 | +| Stage 1 width × height | 768 × 512 | +| Frames / frame rate | 121 / 24 | +| Stage 1 / Stage 2 denoise steps | 8 / 3 | +| Guidance | Fixed positive-only | + +The dimensions passed to the API are the final output dimensions and must be +divisible by 64. Both one- and two-stage requests require `num_frames = 8k+1`. +The distilled recipe fixes both sigma schedules; `num_inference_steps` must be +`8` when supplied. + +## Serving + +Start either checkpoint: + +```bash +vllm serve Lightricks/LTX-2 --omni --stage-init-timeout 600 +``` + +```bash +vllm serve diffusers/LTX-2.3-Diffusers --omni --stage-init-timeout 600 +``` + +Start the distilled checkpoint with its two-stage entry: + +```bash +vllm serve rootonchair/LTX-2-19b-distilled --omni \ + --model-class-name LTX2DistilledPipeline --stage-init-timeout 600 +``` + +The same server handles T2V and I2V. A T2V request using the selected recipe's +default guidance is: + +```bash +curl -X POST http://localhost:8000/v1/videos/sync \ + -F "prompt=A cinematic close-up of ocean waves at golden hour." \ + -F "negative_prompt=worst quality, inconsistent motion, blurry, jittery, distorted" \ + -F "size=768x512" \ + -F "num_frames=121" \ + -F "fps=24" \ + -F "seed=42" \ + -o ltx_t2v.mp4 +``` + +For I2V, provide exactly one initial image: + +```bash +curl -X POST http://localhost:8000/v1/videos/sync \ + -F "prompt=A plush toy astronaut gently waving while the camera slowly pushes in." \ + -F "negative_prompt=worst quality, inconsistent motion, blurry, jittery, distorted" \ + -F "input_reference=@/absolute/path/to/reference.png" \ + -F "size=768x512" \ + -F "num_frames=121" \ + -F "fps=24" \ + -F "seed=42" \ + -o ltx_i2v.mp4 +``` + +Use `image_reference` for a URL or JSON-safe image reference. Do not provide it +together with `input_reference`. + +## Guidance + +LTX supports independent video/audio CFG, spatio-temporal guidance (STG), +cross-modality guidance, and rescaling: + +| Parameter | Default | Effect | Alias | +|---|---:|---|---| +| `video_cfg_scale` | 3.0 | Video text CFG; `1.0` disables it | `video_cfg_guidance_scale` | +| `audio_cfg_scale` | 7.0 | Audio text CFG; `1.0` disables it | `audio_cfg_guidance_scale` | +| `video_stg_scale` | 1.0 | Video STG; `0.0` disables it | `video_stg_guidance_scale` | +| `audio_stg_scale` | 1.0 | Audio STG; `0.0` disables it | `audio_stg_guidance_scale` | +| `video_modality_scale` | 3.0 | Audio-to-video guidance; `1.0` disables it | `a2v_guidance_scale` | +| `audio_modality_scale` | 3.0 | Video-to-audio guidance; `1.0` disables it | `v2a_guidance_scale` | +| `video_rescale_scale` | 0.7 | Video guidance rescale; `0.0` disables it | — | +| `audio_rescale_scale` | 0.7 | Audio guidance rescale; `0.0` disables it | — | +| `video_stg_blocks` | `[29]` / `[28]` | Perturbed video transformer blocks | — | +| `audio_stg_blocks` | `[29]` / `[28]` | Perturbed audio transformer blocks | — | + +The STG block defaults are shown as LTX-2 / LTX-2.3. A canonical name wins +over its alias. Explicit top-level `guidance_scale` wins over both CFG fields; +omit it for independent video/audio CFG. + +Pass these fields in online `extra_params` or offline `--extra-body`: + +```bash +# Add to the curl request above +-F 'extra_params={"video_cfg_scale":3.0,"audio_cfg_scale":7.0}' + +# Add to text_to_video.py or image_to_video.py +--extra-body '{"video_cfg_scale":3.0,"audio_cfg_scale":7.0}' +``` + +### Python API + +Normal per-request values belong in `OmniDiffusionSamplingParams`. Pass the +resulting `DiffusionRequestBatch` as the only positional argument: + +```python +from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch +from vllm_omni.inputs.data import OmniDiffusionSamplingParams + +params = OmniDiffusionSamplingParams( + width=768, + height=512, + num_frames=121, + frame_rate=24.0, + num_inference_steps=30, + seed=42, + extra_args={"video_cfg_scale": 3.0, "audio_cfg_scale": 7.0}, +) +req = DiffusionRequestBatch([ + OmniDiffusionRequest( + prompt={"prompt": "Cherry blossoms moving in a light breeze"}, + sampling_params=params, + request_id="ltx-example", + ) +]) + +t2v_output = pipe(req) +i2v_output = pipe(req, image=image) +``` + +Direct keyword arguments are low-level fallbacks, for example +`pipe(req, prompt=prompt, width=768)`. Request fields take precedence unless +noted below. + +### Complete `forward` Surface + +| Argument | Type/default | Meaning and constraints | +|---|---|---| +| `req` | `DiffusionRequestBatch`, required | Only positional argument; contains prompts and per-request sampling parameters. | +| `image` | image or batch, `None` | Direct value wins over request images; no image selects T2V. I2V accepts one image per prompt, and a batch cannot mix T2V/I2V. | +| `prompt` | string or list, `None` | Positive-text fallback; request prompts win. Mutually exclusive with `prompt_embeds`. | +| `negative_prompt` | string or list, `None` | Fallback after request values and before the recipe default; mutually exclusive with negative embeddings. | +| `height` | `int`, `None` | Request → direct value → recipe default; divisible by 32 one-stage or 64 distilled. | +| `width` | `int`, `None` | Same precedence and alignment as `height`. | +| `num_frames` | `int`, `None` | Request → direct value → recipe default; must be `8k+1` and also determines audio duration with `frame_rate`. | +| `frame_rate` | `float`, `None` | Request `frame_rate` → request `fps` → direct value → recipe default. | +| `num_inference_steps` | `int`, `None` | Request → direct value → recipe default; minimum 2 one-stage, fixed at 8 for distilled Stage 1. Custom one-stage `sigmas` determine actual steps. | +| `sigmas` | list of float, `None` | One-stage only. Request values win; every request in a fused batch must use the same schedule. | +| `timesteps` | list of int, `None` | Compatibility slot; LTX accepts only `None`. Use `sigmas`. | +| `guidance_scale` | `float`, `None` | One-stage common video/audio CFG fallback; an explicit request value wins. Distilled two-stage uses fixed positive-only guidance. | +| `guidance_rescale` | `float`, `None` | Accepts only `None` or `0.0`; use the modality rescale fields. | +| `noise_scale` | `float`, `0.0` | Compatibility slot; LTX accepts only `0.0`. | +| `num_videos_per_prompt` | `int`, `1` | Output-count fallback; positive request `num_outputs_per_prompt` wins. | +| `generator` | generator or list, `None` | Explicit RNG; otherwise request generators/seeds are collated. Lists must match the effective output batch. | +| `latents` | tensor, `None` | Request tensors win. One-stage accepts packed `[B, S, C]` and validated unpacked video layouts. | +| `audio_latents` | tensor, `None` | One-stage initial audio latents; request tensors win and are collated. | +| `prompt_embeds` | tensor, `None` | Precomputed positive conditioning; requires `prompt_attention_mask` and cannot accompany `prompt`. | +| `negative_prompt_embeds` | tensor, `None` | One-stage precomputed negative conditioning; requires its mask and cannot accompany `negative_prompt`. | +| `prompt_attention_mask` | tensor, `None` | Mask for positive embeddings; request `prompt_attention_mask` or `attention_mask` wins. | +| `negative_prompt_attention_mask` | tensor, `None` | Mask for negative embeddings; request negative mask fields win. | +| `decode_timestep` | float or list, `0.0` | Video-VAE decode timestep; request value wins. Lists may match 1, prompt batch, or output batch. | +| `decode_noise_scale` | float or list, `None` | Same list rules; request value wins. Defaults to `decode_timestep`. | +| `output_type` | string, `"np"` | Request value wins. `"np"` decodes output; `"latent"` skips VAE/vocoder decode. | +| `return_dict` | `bool`, `True` | Compatibility slot; only `True` is accepted. | +| `attention_kwargs` | dict, `None` | Per-call values are unsupported; configure attention at engine startup. | +| `max_sequence_length` | `int`, `None` | Request → direct value → tokenizer limit. | + +Request-object naming differs only in a few places: `num_videos_per_prompt` +maps to `num_outputs_per_prompt`; images and prompt text/embeddings live in the +request prompt payload; LTX guidance fields live in sampling `extra_args`. + +### Recipe-Specific Request Capabilities + +| Override | One-stage | Distilled two-stage | +|---|---|---| +| Guidance | Supported | Fixed positive-only | +| Negative prompt/embeddings | Supported | Rejected | +| `num_inference_steps` | Supported | Fixed at 8 for Stage 1; Stage 2 uses 3 | +| Custom `sigmas` | Supported | Rejected; both phases use fixed schedules | +| Video/audio latents | Supported | Rejected | + +These capability checks apply equally to direct `forward` keywords and +values in `OmniDiffusionSamplingParams`; unsupported values fail instead of +being silently ignored. + +### Custom Sigma Schedules + +One-stage Python requests may set final scheduler boundaries directly: + +```python +params = OmniDiffusionSamplingParams(sigmas=[1.0, 0.75, 0.5, 0.25]) +``` + +Each nonterminal value produces one denoise step, and a terminal `0.0` is +appended when omitted. This schedule overrides `num_inference_steps`. All +requests in a fused batch must use the same list. The video form API and +bundled offline CLI do not currently expose `sigmas`. + +### Constraints + +- LTX rejects non-default `timesteps`, `flow_shift`, `guidance_rescale`, + `noise_scale`, `attention_kwargs`, and `return_dict=False`. Use final + `sigmas`, modality rescale fields, startup attention configuration, and the + standard `DiffusionOutput` respectively. +- Fused one-stage requests must resolve to identical LTX guidance and sigma + schedules. Keep concurrent requests guidance-homogeneous or use + `--max-num-seqs 1`. +- `--cfg-parallel-size 2` supports only the CFG-only plan: disable STG and + modality guidance and set both rescale fields to `0.0`. + +## Operational Notes + +- LTX-2 one-stage previously loaded and peaked at about 73.5 GiB on one H200 + 141GB; remeasure on the deployed commit and hardware. +- LTX-2.3 includes a 22B transformer, Gemma encoder, two VAEs, and a vocoder. + Start on a 96GB-class GPU or use CPU/layerwise offload on smaller devices. +- The distilled upsampler uses the configured pipeline dtype and participates + in component discovery/device placement; it remains resident during denoise + offload because it is used only at the phase boundary. +- The output audio sample rate comes from the loaded components and is not a + request parameter. +- For benchmarks, use `tests/dfx/perf/tests/test_ltx2_vllm_omni.json` with + `tests/dfx/perf/scripts/run_diffusion_benchmark.py`. +- Current two-stage support is limited to the distilled checkpoint; official + LTX two-stage/HQ execution remains out of scope. ## References -- Model: -- User guide: - [`docs/user_guide/examples/online_serving/text_to_video.md`](../../docs/user_guide/examples/online_serving/text_to_video.md) -- Example guide: - [`examples/online_serving/text_to_video/README.md`](../../examples/online_serving/text_to_video/README.md) - [`examples/online_serving/image_to_video/README.md`](../../examples/online_serving/image_to_video/README.md) - -## Hardware Support - -## GPU - -### 1x H200 141GB (Text-to-Video) - -#### Environment - -- OS: Ubuntu 22.04.5 LTS -- Python: 3.12+ -- Driver / runtime: NVIDIA CUDA environment with H200 141GB GPU -- vLLM version: Match the repository requirements from your current checkout -- vLLM-Omni version or commit: Use the commit you are deploying from - -#### Command - -```bash -vllm serve Lightricks/LTX-2 \ - --omni \ - --model-class-name LTX2Pipeline -``` - -#### Verification - -```bash -curl http://localhost:8000/health - -curl -X POST http://localhost:8000/v1/videos \ - -F "prompt=A cinematic close-up of ocean waves at golden hour." \ - -F "negative_prompt=worst quality, inconsistent motion, blurry, jittery, distorted" \ - -F "width=512" \ - -F "height=768" \ - -F "num_frames=121" \ - -F "fps=24" \ - -F "num_inference_steps=40" \ - -F "guidance_scale=4.0" \ - -o ltx2_t2v_h200.mp4 -``` - -#### Notes - -- Memory usage: Model loads at ~73.5GB, peaks at ~73.5 GiB during inference. -- Key flags: - - `--model-class-name LTX2Pipeline` selects the text-to-video pipeline explicitly. -- Known limitations: - - This recipe is tested on 1x H200 validation; add measured latency and VRAM after running it on your target machine. - -### 1x H200 141GB (Image-to-Video) - -#### Environment - -- OS: Ubuntu 22.04.5 LTS -- Python: 3.12+ -- Driver / runtime: NVIDIA CUDA environment with H200 141GB GPU -- vLLM version: Match the repository requirements from your current checkout -- vLLM-Omni version or commit: Use the commit you are deploying from - -#### Command - -```bash -vllm serve Lightricks/LTX-2 \ - --omni \ - --model-class-name LTX2ImageToVideoPipeline -``` - -#### Verification - -```bash -curl http://localhost:8000/health - -curl -X POST http://localhost:8000/v1/videos/sync \ - -F "prompt=A plush toy astronaut gently waving while the camera slowly pushes in." \ - -F "input_reference=@/absolute/path/to/reference.png" \ - -F "width=512" \ - -F "height=768" \ - -F "num_frames=121" \ - -F "fps=24" \ - -F "num_inference_steps=40" \ - -F "guidance_scale=4.0" \ - -o ltx2_i2v_h200.mp4 -``` - -#### Notes - -- Memory usage: Model loads at ~73.5GB, peaks at ~73.5 GiB during inference. -- Key flags: - - `--model-class-name LTX2ImageToVideoPipeline` switches the same `Lightricks/LTX-2` weights into image-to-video mode. - - `input_reference=@...` uploads the reference image directly -- Known limitations: - - Use `image_reference` when you want to pass a URL or JSON-safe image reference -instead of uploading a file. - - Do not send `input_reference` and `image_reference` in the same request. - - This recipe is tested on 1x H200 validation; add measured latency and VRAM after running it on your target machine. +- +- +- +- [Online video generation](../../docs/user_guide/examples/online_serving/text_to_video.md) +- [Request batching](../../docs/user_guide/diffusion/request_batching.md) +- [T2V offline example](../../examples/offline_inference/text_to_video/text_to_video.md) +- [I2V offline example](../../examples/offline_inference/image_to_video/README.md) diff --git a/recipes/README.md b/recipes/README.md index cc1ac4920..1d89f0ef1 100644 --- a/recipes/README.md +++ b/recipes/README.md @@ -46,8 +46,7 @@ recipes/ | [`inclusionAI/Ming-omni-tts.md`](./inclusionAI/Ming-omni-tts.md) | Offline + online dense Ming TTS/audio generation | 1x H100 80GB / 1x AMD MI300X (ROCm 7.2) | | [`IndexTeam/IndexTTS-2.md`](./IndexTeam/IndexTTS-2.md) | Online serving for voice-cloned TTS with optional emotion control | 1x L4 24GB or larger CUDA GPU | | [`krea/Krea-2.md`](./krea/Krea-2.md) | Text-to-image (Turbo + Raw), offline + online, with LoRA | 1x H100 80GB | -| [`LTX/LTX-2.md`](./LTX/LTX-2.md) | Text-to-video and image-to-video serving | 1x H200 141GB | -| [`LTX/LTX-2.3.md`](./LTX/LTX-2.3.md) | Text-to-video with audio generation (22B) | 1x GPU (96GB VRAM) | +| [`LTX/LTX-2.md`](./LTX/LTX-2.md) | LTX-2/LTX-2.3 text-to-video and image-to-video with synchronized audio | H200 141GB / 96GB-class GPU | | [`MammothModa2/MammothModa2.md`](./MammothModa2/MammothModa2.md) | Preview and Dev text-to-image (AR → DiT); Dev text/image understanding | Preview: 1x L40S 48GB / 1x ≥40GB GPU; Dev: 1x NVIDIA GPU with sufficient cache headroom | | [`mistralai/Voxtral-TTS.md`](./mistralai/Voxtral-TTS.md) | Online serving for TTS | 1x RTX 4090 24GB | | [`Robbyant/LingBot-Video.md`](./Robbyant/LingBot-Video.md) | Native dense and MoE text-to-video serving | Dense: 1x L20X; MoE: 1x L20X (~67.7 GiB peak) | diff --git a/tests/diffusion/models/dmd2/test_dmd2_request_sanitization.py b/tests/diffusion/models/dmd2/test_dmd2_request_sanitization.py index 5f8de588e..5fe027a20 100644 --- a/tests/diffusion/models/dmd2/test_dmd2_request_sanitization.py +++ b/tests/diffusion/models/dmd2/test_dmd2_request_sanitization.py @@ -8,7 +8,6 @@ import torch from vllm_omni.diffusion.models.flux.pipeline_flux import FluxDMD2Pipeline, FluxPipeline from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import ( LTX2I2VDMD2Pipeline, - LTX2ImageToVideoPipeline, LTX2Pipeline, LTX2T2VDMD2Pipeline, ) @@ -25,7 +24,7 @@ _DMD2_BASE = { WanT2VDMD2Pipeline: Wan22Pipeline, WanI2VDMD2Pipeline: Wan22I2VPipeline, LTX2T2VDMD2Pipeline: LTX2Pipeline, - LTX2I2VDMD2Pipeline: LTX2ImageToVideoPipeline, + LTX2I2VDMD2Pipeline: LTX2Pipeline, FluxDMD2Pipeline: FluxPipeline, QwenImageDMD2Pipeline: QwenImagePipeline, } diff --git a/tests/diffusion/models/dmd2/test_dmd2_scheduler.py b/tests/diffusion/models/dmd2/test_dmd2_scheduler.py index a395af959..4df65f1fe 100644 --- a/tests/diffusion/models/dmd2/test_dmd2_scheduler.py +++ b/tests/diffusion/models/dmd2/test_dmd2_scheduler.py @@ -8,7 +8,6 @@ import torch from vllm_omni.diffusion.models.flux.pipeline_flux import FluxDMD2Pipeline, FluxPipeline from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import ( LTX2I2VDMD2Pipeline, - LTX2ImageToVideoPipeline, LTX2Pipeline, LTX2T2VDMD2Pipeline, ) @@ -27,7 +26,7 @@ _DMD2_BASE = { WanT2VDMD2Pipeline: Wan22Pipeline, WanI2VDMD2Pipeline: Wan22I2VPipeline, LTX2T2VDMD2Pipeline: LTX2Pipeline, - LTX2I2VDMD2Pipeline: LTX2ImageToVideoPipeline, + LTX2I2VDMD2Pipeline: LTX2Pipeline, FluxDMD2Pipeline: FluxPipeline, QwenImageDMD2Pipeline: QwenImagePipeline, } diff --git a/tests/diffusion/models/ltx2/test_ltx2_conditioning.py b/tests/diffusion/models/ltx2/test_ltx2_conditioning.py index 43737f8c1..6b4ec70ce 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_conditioning.py +++ b/tests/diffusion/models/ltx2/test_ltx2_conditioning.py @@ -7,6 +7,7 @@ from types import SimpleNamespace import pytest import torch +from PIL import Image pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -16,17 +17,31 @@ def _make_ltx23_request_pipe(cls): torch.nn.Module.__init__(pipe) pipe.device = torch.device("cpu") pipe.tokenizer_max_length = 99 + pipe.vae_spatial_compression_ratio = 32 + pipe.vae_temporal_compression_ratio = 8 return pipe class TestLTXImageToVideoForwardStages: - def test_forward_resolves_request_image_and_delegates_to_shared_forward_impl(self): - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23ImageToVideoPipeline + def test_i2v_pil_preprocessing_matches_official_center_crop(self): + from vllm_omni.diffusion.models.ltx2.ltx2_conditioning import _preprocess_i2v_pil_images + + pixels = torch.zeros(4, 8, 3, dtype=torch.uint8) + pixels[:, :, 0] = torch.arange(8, dtype=torch.uint8) + image = Image.fromarray(pixels.numpy()) + + actual = _preprocess_i2v_pil_images(image, height=4, width=4) + expected = pixels[:, 2:6].permute(2, 0, 1).unsqueeze(0).float() / 127.5 - 1.0 + + torch.testing.assert_close(actual, expected) + + def test_forward_resolves_request_image_and_delegates_to_shared_recipe_runtime(self): + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch from vllm_omni.inputs.data import OmniDiffusionSamplingParams - pipe = _make_ltx23_request_pipe(LTX23ImageToVideoPipeline) + pipe = _make_ltx23_request_pipe(LTX2Pipeline) image = torch.zeros(3, 8, 8) req = DiffusionRequestBatch( [ @@ -48,27 +63,45 @@ class TestLTXImageToVideoForwardStages: ) seen = {} - def fake_forward_impl(req_arg, request_inputs, **kwargs): + def fake_run_recipe(req_arg, request_inputs, **kwargs): seen["req"] = req_arg seen["request_inputs"] = request_inputs seen["kwargs"] = kwargs return ["i2v-delegated"] - object.__setattr__(pipe, "_forward_impl", fake_forward_impl) + object.__setattr__(pipe, "_run_recipe", fake_run_recipe) - output = pipe.forward(req, noise_scale=0.5) + output = pipe.forward(req) assert output == ["i2v-delegated"] assert seen["req"] is req assert seen["request_inputs"].prompt == ["make the image move"] assert seen["request_inputs"].negative_prompt == ["jitter"] assert seen["kwargs"]["image"] is image - assert seen["kwargs"]["noise_scale"] == 0.5 + assert seen["kwargs"]["request_sigmas"] is None + + def test_unified_entry_selects_t2v_without_images_and_rejects_mixed_batches(self): + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline + + pipe = object.__new__(LTX2Pipeline) + request_inputs = SimpleNamespace(latents=None) + t2v_req = SimpleNamespace(prompts=["text only", {"prompt": "also text only"}]) + + assert pipe._resolve_request_image(t2v_req, None, request_inputs) is None + + mixed_req = SimpleNamespace( + prompts=[ + "text only", + {"prompt": "image conditioned", "multi_modal_data": {"image": torch.zeros(3, 8, 8)}}, + ] + ) + with pytest.raises(ValueError, match="cannot mix text-to-video and image-to-video"): + pipe._resolve_request_image(mixed_req, None, request_inputs) def test_denoise_timestep_kwargs_masks_video_only(self): - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23ImageToVideoPipeline + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline - pipe = object.__new__(LTX23ImageToVideoPipeline) + pipe = object.__new__(LTX2Pipeline) ts = torch.tensor([2.0, 4.0]) denoise_ctx = SimpleNamespace( conditioning_mask=torch.tensor([[1.0, 0.0]]), @@ -79,30 +112,32 @@ class TestLTXImageToVideoForwardStages: ts, SimpleNamespace(cfg_parallel_ready=False), denoise_ctx, + video_token_count=2, + audio_token_count=1, ) torch.testing.assert_close(kwargs["timestep"], torch.tensor([[0.0, 2.0], [4.0, 0.0]])) - torch.testing.assert_close(kwargs["audio_timestep"], ts) + torch.testing.assert_close(kwargs["audio_timestep"], ts[:, None]) torch.testing.assert_close(kwargs["sigma"], ts) class TestLTXImageToVideoConditioning: def test_ltx23_i2v_supports_image_input(self): - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23ImageToVideoPipeline + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline - assert LTX23ImageToVideoPipeline.support_image_input is True + assert LTX2Pipeline.support_image_input is True def test_ltx23_i2v_rejects_multi_image_prompt_list(self): - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23ImageToVideoPipeline + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline image = object() - assert LTX23ImageToVideoPipeline._resolve_single_prompt_image([image]) is image + assert LTX2Pipeline._resolve_single_prompt_image([image]) is image with pytest.raises(ValueError, match="exactly one image per prompt"): - LTX23ImageToVideoPipeline._resolve_single_prompt_image([object(), object()]) + LTX2Pipeline._resolve_single_prompt_image([object(), object()]) def test_ltx23_i2v_additional_image_resolution_is_tensor_safe(self): - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23ImageToVideoPipeline + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline image = torch.zeros(1, 3, 4, 4) additional = { @@ -111,19 +146,24 @@ class TestLTXImageToVideoConditioning: "image": torch.ones_like(image), } - assert LTX23ImageToVideoPipeline._resolve_additional_image(additional) is image + assert LTX2Pipeline._resolve_additional_image(additional) is image def test_ltx23_i2v_packed_latents_are_not_noised(self, monkeypatch): import vllm_omni.diffusion.models.ltx2.ltx2_conditioning as ltx2_conditioning import vllm_omni.diffusion.models.ltx2.ltx2_latents as ltx2_latents - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23ImageToVideoPipeline + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline - pipe = object.__new__(LTX23ImageToVideoPipeline) + pipe = object.__new__(LTX2Pipeline) torch.nn.Module.__init__(pipe) pipe.vae_spatial_compression_ratio = 1 pipe.vae_temporal_compression_ratio = 1 pipe.transformer_spatial_patch_size = 1 pipe.transformer_temporal_patch_size = 1 + object.__setattr__( + pipe, + "_encode_i2v_image_latents", + lambda *_args, **_kwargs: torch.tensor([[[[[40.0]]], [[[41.0]]]]]), + ) def fake_randn_tensor(shape, generator=None, device=None, dtype=None): raise AssertionError("packed I2V latents should not be noised") @@ -134,7 +174,7 @@ class TestLTXImageToVideoConditioning: latents = torch.tensor([[[10.0, 11.0], [20.0, 21.0], [30.0, 31.0]]]) out, conditioning_mask = pipe.prepare_latents( - image=None, + image=torch.zeros(1, 3, 1, 1), batch_size=1, num_channels_latents=2, height=1, @@ -147,14 +187,14 @@ class TestLTXImageToVideoConditioning: ) torch.testing.assert_close(conditioning_mask, torch.tensor([[1.0, 0.0, 0.0]])) - torch.testing.assert_close(out, latents) + torch.testing.assert_close(out, torch.tensor([[[40.0, 41.0], [20.0, 21.0], [30.0, 31.0]]])) def test_ltx23_i2v_5d_latents_noise_preserves_conditioning_frame(self, monkeypatch): import vllm_omni.diffusion.models.ltx2.ltx2_conditioning as ltx2_conditioning import vllm_omni.diffusion.models.ltx2.ltx2_latents as ltx2_latents - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23ImageToVideoPipeline + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline - pipe = object.__new__(LTX23ImageToVideoPipeline) + pipe = object.__new__(LTX2Pipeline) torch.nn.Module.__init__(pipe) pipe.vae_spatial_compression_ratio = 1 pipe.vae_temporal_compression_ratio = 1 @@ -165,8 +205,16 @@ class TestLTXImageToVideoConditioning: latents_std=torch.ones(2), config=SimpleNamespace(scaling_factor=1.0), ) + object.__setattr__( + pipe, + "_encode_i2v_image_latents", + lambda *_args, **_kwargs: torch.tensor([[[[[40.0]]], [[[41.0]]]]]), + ) + + sampled_shapes = [] def fake_randn_tensor(shape, generator=None, device=None, dtype=None): + sampled_shapes.append(tuple(shape)) return torch.ones(shape, device=device, dtype=dtype) monkeypatch.setattr(ltx2_conditioning, "randn_tensor", fake_randn_tensor) @@ -175,7 +223,7 @@ class TestLTXImageToVideoConditioning: latents = torch.tensor([[[[[10.0]], [[20.0]], [[30.0]]], [[[11.0]], [[21.0]], [[31.0]]]]]) out, conditioning_mask = pipe.prepare_latents( - image=None, + image=torch.zeros(1, 3, 1, 1), batch_size=1, num_channels_latents=2, height=1, @@ -187,20 +235,64 @@ class TestLTXImageToVideoConditioning: latents=latents, ) + torch.testing.assert_close(conditioning_mask, torch.tensor([[1.0, 0.0, 0.0]])) + torch.testing.assert_close(out, torch.tensor([[[40.0, 41.0], [1.0, 1.0], [1.0, 1.0]]])) + assert sampled_shapes == [(1, 3, 2)] + + def test_ltx23_i2v_image_noise_is_sampled_after_packing(self, monkeypatch): + import vllm_omni.diffusion.models.ltx2.ltx2_conditioning as ltx2_conditioning + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline + + pipe = object.__new__(LTX2Pipeline) + torch.nn.Module.__init__(pipe) + pipe.vae_spatial_compression_ratio = 1 + pipe.vae_temporal_compression_ratio = 1 + pipe.transformer_spatial_patch_size = 1 + pipe.transformer_temporal_patch_size = 1 + pipe.vae = SimpleNamespace( + encode=lambda image: image, + latents_mean=torch.zeros(2), + latents_std=torch.ones(2), + ) + monkeypatch.setattr( + ltx2_conditioning, + "retrieve_latents", + lambda *_args, **_kwargs: torch.tensor([[[[[10.0]]], [[[11.0]]]]]), + ) + sampled_shapes = [] + + def fake_randn_tensor(shape, generator=None, device=None, dtype=None): + sampled_shapes.append(tuple(shape)) + return torch.ones(shape, device=device, dtype=dtype) + + monkeypatch.setattr(ltx2_conditioning, "randn_tensor", fake_randn_tensor) + + out, conditioning_mask = pipe.prepare_latents( + image=torch.zeros(1, 3, 1, 1), + batch_size=1, + num_channels_latents=2, + height=1, + width=1, + num_frames=3, + noise_scale=1.0, + dtype=torch.float32, + device=torch.device("cpu"), + ) + + assert sampled_shapes == [(1, 3, 2)] torch.testing.assert_close(conditioning_mask, torch.tensor([[1.0, 0.0, 0.0]])) torch.testing.assert_close(out, torch.tensor([[[10.0, 11.0], [1.0, 1.0], [1.0, 1.0]]])) def test_ltx23_i2v_video_step_preserves_conditioning_frame(self): - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23ImageToVideoPipeline + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline - pipe = object.__new__(LTX23ImageToVideoPipeline) + pipe = object.__new__(LTX2Pipeline) torch.nn.Module.__init__(pipe) pipe.transformer_spatial_patch_size = 1 pipe.transformer_temporal_patch_size = 1 class FakeScheduler: - def step(self, noise_pred, t, latents, return_dict=False): - return (latents + noise_pred + t,) + sigmas = torch.tensor([1.0, 0.5]) pipe.scheduler = FakeScheduler() latents = torch.tensor([[[1.0], [2.0], [3.0]]]) @@ -209,11 +301,103 @@ class TestLTXImageToVideoConditioning: out = pipe._step_video_latents_i2v( noise_pred, latents, - torch.tensor(0.5), + 0, latent_num_frames=3, latent_height=1, latent_width=1, ) torch.testing.assert_close(out[:, :1], latents[:, :1]) - torch.testing.assert_close(out[:, 1:], latents[:, 1:] + noise_pred[:, 1:] + 0.5) + torch.testing.assert_close(out[:, 1:], latents[:, 1:] - 0.5 * noise_pred[:, 1:]) + + def test_shared_step_adapter_uses_official_euler_for_t2v(self): + from vllm_omni.diffusion.models.ltx2.ltx2_denoise import LTXVideoAudioStepAdapter + + pipeline = SimpleNamespace(scheduler=SimpleNamespace(sigmas=torch.tensor([1.0, 0.5]))) + adapter = LTXVideoAudioStepAdapter( + pipeline, + SimpleNamespace(sigmas=torch.tensor([1.0, 0.25])), + latent_num_frames=1, + latent_height=1, + latent_width=1, + image_conditioned=False, + ) + video = torch.tensor([[[2.0]]]) + audio = torch.tensor([[[4.0]]]) + + ((video, audio),) = adapter.step( + (torch.tensor([[[2.0]]]), torch.tensor([[[4.0]]])), + (torch.tensor(1.0), torch.tensor(1.0)), + (video, audio), + ) + + torch.testing.assert_close(video, torch.tensor([[[1.0]]])) + torch.testing.assert_close(audio, torch.tensor([[[1.0]]])) + + def test_shared_step_adapter_uses_official_euler_for_i2v_audio(self): + from vllm_omni.diffusion.models.ltx2.ltx2_denoise import LTXVideoAudioStepAdapter + + class FakePipeline: + scheduler = object() + + def __init__(self): + self.step_indices = [] + + def _step_video_latents_i2v( + self, + noise_pred, + latents, + step_index, + latent_num_frames, + latent_height, + latent_width, + ): + del noise_pred, latent_num_frames, latent_height, latent_width + self.step_indices.append(step_index) + return latents + + pipeline = FakePipeline() + adapter = LTXVideoAudioStepAdapter( + pipeline, + SimpleNamespace(sigmas=torch.tensor([1.0, 0.5, 0.0])), + latent_num_frames=1, + latent_height=1, + latent_width=1, + image_conditioned=True, + ) + video = torch.tensor([[[1.0]]]) + audio = torch.tensor([[[2.0]]]) + video_velocity = torch.zeros_like(video) + audio_velocity = torch.tensor([[[4.0]]]) + + ((video, audio),) = adapter.step( + (video_velocity, audio_velocity), + (torch.tensor(1.0), torch.tensor(1.0)), + (video, audio), + ) + torch.testing.assert_close(audio, torch.tensor([[[0.0]]])) + + ((video, audio),) = adapter.step( + (video_velocity, audio_velocity), + (torch.tensor(0.5), torch.tensor(0.5)), + (video, audio), + ) + torch.testing.assert_close(audio, torch.tensor([[[-2.0]]])) + assert pipeline.step_indices == [0, 1] + + def test_i2v_guidance_uses_zero_sigma_for_conditioned_tokens(self): + from vllm_omni.diffusion.models.ltx2.ltx2_denoise import LTXDenoiseContext + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline + + pipe = object.__new__(LTX2Pipeline) + denoise_ctx = LTXDenoiseContext( + latents=torch.empty(1, 3, 1), + audio_latents=torch.empty(1, 1, 1), + video_coords=torch.empty(1), + audio_coords=torch.empty(1), + conditioning_mask=torch.tensor([[1.0, 0.0, 0.0]]), + ) + + actual = pipe._video_guidance_model_sigma(torch.tensor(0.75), denoise_ctx) + + torch.testing.assert_close(actual, torch.tensor([[[0.0], [0.75], [0.75]]])) diff --git a/tests/diffusion/models/ltx2/test_ltx2_latents.py b/tests/diffusion/models/ltx2/test_ltx2_latents.py index 74cad09d1..36f7e9687 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_latents.py +++ b/tests/diffusion/models/ltx2/test_ltx2_latents.py @@ -4,7 +4,7 @@ import pytest import torch from vllm_omni.diffusion.models.ltx2 import ltx2_latents as latent_ops -from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline, LTX23Pipeline +from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -23,8 +23,51 @@ def _make_pipeline(pipeline_cls, sequence_parallel_size: int = 1): return pipeline +def test_prepare_video_latents_samples_directly_in_packed_token_space(): + pipeline = _make_pipeline(LTX2Pipeline) + pipeline.vae_spatial_compression_ratio = 8 + pipeline.vae_temporal_compression_ratio = 8 + pipeline.transformer_spatial_patch_size = 2 + pipeline.transformer_temporal_patch_size = 1 + + expected_generator = torch.Generator().manual_seed(42) + expected = torch.randn((1, 32, 16), generator=expected_generator) + actual = pipeline.prepare_latents( + batch_size=1, + num_channels_latents=4, + height=64, + width=64, + num_frames=9, + dtype=torch.float32, + device=torch.device("cpu"), + generator=torch.Generator().manual_seed(42), + ) + + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + +def test_prepare_audio_latents_samples_directly_in_packed_token_space(): + pipeline = _make_pipeline(LTX2Pipeline) + + expected_generator = torch.Generator().manual_seed(42) + expected = torch.randn((1, 3, 32), generator=expected_generator) + actual, original_num_frames, padded_num_frames = pipeline.prepare_audio_latents( + batch_size=1, + num_channels_latents=2, + num_mel_bins=64, + audio_latent_length=3, + dtype=torch.float32, + device=torch.device("cpu"), + generator=torch.Generator().manual_seed(42), + ) + + assert original_num_frames == 3 + assert padded_num_frames == 3 + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + def test_prepare_audio_latents_pads_generated_dummy_length_for_sp(): - pipeline = _make_pipeline(LTX23Pipeline, sequence_parallel_size=2) + pipeline = _make_pipeline(LTX2Pipeline, sequence_parallel_size=2) latents, original_num_frames, padded_num_frames = pipeline.prepare_audio_latents( batch_size=1, @@ -66,8 +109,8 @@ def test_unpad_audio_latents_restores_original_frames_before_unpack(): padded = torch.cat([original, torch.full((1, 2, 4), 999.0)], dim=1) unpadded = latent_ops.unpad_audio_latents(padded, 10) - unpacked = latent_ops.unpack_audio_latents(unpadded, latent_length=10, num_mel_bins=2) - expected = latent_ops.unpack_audio_latents(original, latent_length=10, num_mel_bins=2) + unpacked = latent_ops.unpack_audio_latents(unpadded, num_mel_bins=2) + expected = latent_ops.unpack_audio_latents(original, num_mel_bins=2) assert unpacked.shape == (1, 2, 10, 2) assert not (unpacked == 999.0).any() @@ -75,7 +118,8 @@ def test_unpad_audio_latents_restores_original_frames_before_unpack(): def test_prepare_audio_latents_accepts_already_padded_4d_latents_for_sp(): - pipeline = _make_pipeline(LTX23Pipeline, sequence_parallel_size=4) + pipeline = _make_pipeline(LTX2Pipeline, sequence_parallel_size=4) + pipeline.preserve_sp_padded_audio_duration = True latents = torch.arange(96, dtype=torch.float32).view(1, 2, 12, 4) audio_latent_length = pipeline._resolve_audio_latent_length(10, latents) @@ -97,7 +141,7 @@ def test_prepare_audio_latents_accepts_already_padded_4d_latents_for_sp(): def test_resolve_audio_latent_length_preserves_legacy_4d_shape_inference(): - pipeline = _make_pipeline(LTX23Pipeline, sequence_parallel_size=4) + pipeline = _make_pipeline(LTX2Pipeline, sequence_parallel_size=4) latents = torch.zeros(1, 2, 13, 4) audio_latent_length = pipeline._resolve_audio_latent_length(10, latents) @@ -106,7 +150,7 @@ def test_resolve_audio_latent_length_preserves_legacy_4d_shape_inference(): def test_prepare_audio_latents_rejects_incompatible_provided_length(): - pipeline = _make_pipeline(LTX23Pipeline, sequence_parallel_size=4) + pipeline = _make_pipeline(LTX2Pipeline, sequence_parallel_size=4) latents = torch.zeros(1, 11, 4) with pytest.raises(ValueError, match="incompatible audio frame count"): diff --git a/tests/diffusion/models/ltx2/test_ltx2_parallel.py b/tests/diffusion/models/ltx2/test_ltx2_parallel.py index 0c6b12fbe..92077a229 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_parallel.py +++ b/tests/diffusion/models/ltx2/test_ltx2_parallel.py @@ -15,9 +15,9 @@ class TestCFGParallelHelpers: """Test LTX-2.3 CFG helper math without loading model weights.""" def test_combine_cfg_noise_matches_x0_space_formula(self): - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23Pipeline + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline - pipe = object.__new__(LTX23Pipeline) + pipe = object.__new__(LTX2Pipeline) video_sample = torch.tensor([[[1.0, -2.0]]]) audio_sample = torch.tensor([[[0.5, 3.0]]]) video_pos = torch.tensor([[[0.2, -0.3]]]) @@ -50,93 +50,53 @@ class TestCFGParallelHelpers: assert torch.allclose(video_combined, expected_video) assert torch.allclose(audio_combined, expected_audio) - def test_two_rank_cfg_parallel_smoke_uses_rank_local_branch_and_x0_formula(self, monkeypatch): - from vllm_omni.diffusion.models.ltx2 import ltx2_guidance - from vllm_omni.diffusion.models.ltx2 import pipeline_ltx2 as ltx23 - - pipe = object.__new__(ltx23.LTX23Pipeline) - video_sample = torch.tensor([[[1.0, -2.0]]]) - audio_sample = torch.tensor([[[0.5, 3.0, -1.0]]]) - video_pos = torch.tensor([[[0.2, -0.3]]]) - video_neg = torch.tensor([[[-0.4, 0.1]]]) - audio_pos = torch.tensor([[[0.7, -0.2, 0.3]]]) - audio_neg = torch.tensor([[[0.1, 0.4, -0.5]]]) - video_sigma = torch.tensor(0.25) - audio_sigma = torch.tensor(0.5) - scale = 4.0 - - class FakeCfgGroup: - def all_gather(self, tensor, separate_tensors=True): - assert separate_tensors - if tensor.shape == video_pos.shape: - return [video_pos, video_neg] - return [audio_pos, audio_neg] - - monkeypatch.setattr(ltx2_guidance, "get_classifier_free_guidance_world_size", lambda: 2) - monkeypatch.setattr(ltx2_guidance, "get_cfg_group", lambda: FakeCfgGroup()) - - expected_video = ltx2_guidance.combine_velocity_via_x0( - video_sample, - video_pos, - video_neg, - video_sigma, - scale, - ) - expected_audio = ltx2_guidance.combine_velocity_via_x0( - audio_sample, - audio_pos, - audio_neg, - audio_sigma, - scale, + def test_cfg_parallel_rejects_non_cfg_guidance(self): + from vllm_omni.diffusion.models.ltx2.ltx2_guidance import ( + LTX_GUIDANCE_EXECUTOR, + LTXGuidancePlan, ) + from vllm_omni.diffusion.models.ltx2.ltx2_recipes import LTX23_ONE_STAGE_RECIPE - for rank, expected_branch in ((0, "positive"), (1, "negative")): - calls = [] - monkeypatch.setattr(ltx2_guidance, "get_classifier_free_guidance_rank", lambda rank=rank: rank) + plan = LTXGuidancePlan.build(LTX23_ONE_STAGE_RECIPE.request_guidance) - def fake_predict_noise(**kwargs): - calls.append(kwargs["branch"]) - if kwargs["branch"] == "positive": - return video_pos, audio_pos - return video_neg, audio_neg + with pytest.raises(ValueError, match="CFG-only guidance"): + LTX_GUIDANCE_EXECUTOR.validate_cfg_world_size(plan, 2) - object.__setattr__(pipe, "predict_noise", fake_predict_noise) - video_combined, audio_combined = pipe.predict_noise_with_parallel_cfg( - true_cfg_scale=scale, - positive_kwargs={"branch": "positive"}, - negative_kwargs={"branch": "negative"}, - cfg_normalize=False, - video_latents=video_sample, - audio_latents=audio_sample, - video_sigma=video_sigma, - audio_sigma=audio_sigma, - ) + def test_cfg_parallel_dummy_warms_supported_cfg_only_plan(self, monkeypatch): + from vllm_omni.diffusion.models.ltx2 import ltx2_runtime + from vllm_omni.diffusion.models.ltx2.ltx2_recipes import LTX23_ONE_STAGE_RECIPE + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline - assert calls == [expected_branch] - torch.testing.assert_close(video_combined, expected_video) - torch.testing.assert_close(audio_combined, expected_audio) + monkeypatch.setattr(ltx2_runtime, "get_classifier_free_guidance_world_size", lambda: 2) + pipe = object.__new__(LTX2Pipeline) + req = SimpleNamespace(is_dummy_run=lambda: True) + request_inputs = SimpleNamespace(guidance=LTX23_ONE_STAGE_RECIPE.request_guidance) - assert "_cfg_video_latents" not in pipe.__dict__ - assert "_cfg_audio_latents" not in pipe.__dict__ + cfg_parallel_ready = pipe._setup_forward_runtime(req, request_inputs, attention_kwargs=None) + + assert cfg_parallel_ready is True + assert pipe._guidance_plan.names == ("cond", "uncond") + assert pipe._guidance_plan.spec.video.cfg_scale == 3.0 + assert pipe._guidance_plan.spec.audio.cfg_scale == 7.0 class TestCFGParallelForwardPath: """Test the LTX-2.3 CFG-parallel denoising path without loading model weights.""" - @pytest.mark.parametrize("pipeline_cls_name", ["LTX2Pipeline", "LTX23Pipeline"]) - def test_forward_collates_request_prompt_embeds_and_mask_aliases(self, monkeypatch, pipeline_cls_name): - from vllm_omni.diffusion.models.ltx2 import ltx2_pipeline_runtime + def test_forward_collates_request_prompt_embeds_and_mask_aliases(self, monkeypatch): + from vllm_omni.diffusion.models.ltx2 import ltx2_runtime from vllm_omni.diffusion.models.ltx2 import pipeline_ltx2 as ltx23 from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch from vllm_omni.inputs.data import OmniDiffusionSamplingParams - pipeline_cls = getattr(ltx23, pipeline_cls_name) - pipe = object.__new__(pipeline_cls) + pipe = object.__new__(ltx23.LTX2Pipeline) torch.nn.Module.__init__(pipe) pipe.device = torch.device("cpu") pipe.tokenizer_max_length = 4 - monkeypatch.setattr(ltx2_pipeline_runtime, "get_classifier_free_guidance_world_size", lambda: 1) + pipe.vae_spatial_compression_ratio = 32 + pipe.vae_temporal_compression_ratio = 8 + monkeypatch.setattr(ltx2_runtime, "get_classifier_free_guidance_world_size", lambda: 1) class StopAtEncodePromptError(Exception): pass @@ -224,7 +184,7 @@ class TestCFGParallelForwardPath: ("frame_rate_input", "audio_sampling_rate", "expected_frame_rate"), [(1.0, 1, 1.0), (None, 24, 24.0)], ) - def test_forward_cfg_parallel_steps_video_and_audio_scheduler( + def test_forward_cfg_parallel_uses_shared_video_audio_euler_step( self, monkeypatch, cfg_rank, @@ -233,13 +193,13 @@ class TestCFGParallelForwardPath: audio_sampling_rate, expected_frame_rate, ): - from vllm_omni.diffusion.models.ltx2 import ltx2_denoise, ltx2_guidance, ltx2_pipeline_runtime + from vllm_omni.diffusion.models.ltx2 import ltx2_denoise, ltx2_guidance, ltx2_runtime from vllm_omni.diffusion.models.ltx2 import pipeline_ltx2 as ltx23 from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch from vllm_omni.inputs.data import OmniDiffusionSamplingParams - pipe = object.__new__(ltx23.LTX23Pipeline) + pipe = object.__new__(ltx23.LTX2Pipeline) torch.nn.Module.__init__(pipe) pipe.device = torch.device("cpu") pipe.tokenizer_max_length = 1 @@ -278,7 +238,7 @@ class TestCFGParallelForwardPath: return [audio_pos, audio_neg] raise AssertionError(f"Unexpected gathered tensor: {tensor}") - monkeypatch.setattr(ltx2_pipeline_runtime, "get_classifier_free_guidance_world_size", lambda: 2) + monkeypatch.setattr(ltx2_runtime, "get_classifier_free_guidance_world_size", lambda: 2) monkeypatch.setattr(ltx2_guidance, "get_classifier_free_guidance_world_size", lambda: 2) monkeypatch.setattr(ltx2_guidance, "get_classifier_free_guidance_rank", lambda: cfg_rank) monkeypatch.setattr(ltx2_guidance, "get_cfg_group", lambda: FakeCfgGroup()) @@ -314,7 +274,7 @@ class TestCFGParallelForwardPath: def __call__(self, prompt_embeds, prompt_attention_mask, padding_side): assert padding_side == "left" - assert prompt_embeds.shape[0] == 2 + assert prompt_embeds.shape[0] == 1 return prompt_embeds, prompt_embeds, prompt_attention_mask rope_video_fps: list[float] = [] @@ -380,7 +340,16 @@ class TestCFGParallelForwardPath: num_frames=1, frame_rate=frame_rate_input, num_inference_steps=2, - guidance_scale=4.0, + extra_args={ + "video_cfg_scale": 3.0, + "audio_cfg_scale": 7.0, + "video_stg_scale": 0.0, + "audio_stg_scale": 0.0, + "video_modality_scale": 1.0, + "audio_modality_scale": 1.0, + "video_rescale_scale": 0.0, + "audio_rescale_scale": 0.0, + }, latents=video_latents, audio_latents=audio_latents, output_type="latent", @@ -388,35 +357,31 @@ class TestCFGParallelForwardPath: request_id="ltx23-cfg-parallel-forward-test", ) - output = pipe.forward(DiffusionRequestBatch(requests=[req]))[0] + output = pipe.forward(DiffusionRequestBatch(requests=[req]), sigmas=[0.25, 0.25])[0] expected_video_noise = ltx2_guidance.combine_velocity_via_x0( video_latents, video_pos, video_neg, pipe.scheduler.sigmas[0], - 4.0, + 3.0, ) expected_audio_noise = ltx2_guidance.combine_velocity_via_x0( audio_latents, audio_pos, audio_neg, pipe.scheduler.sigmas[0], - 4.0, + 7.0, ) - scheduler_call_names = [call[0] for call in pipe.scheduler.calls] - assert scheduler_call_names == ["video", "audio", "video", "audio"] + assert pipe.scheduler.calls == [] assert len(pipe.transformer.calls) == 2 - torch.testing.assert_close(pipe.scheduler.calls[0][1], expected_video_noise) - torch.testing.assert_close(pipe.scheduler.calls[1][1], expected_audio_noise) - torch.testing.assert_close(pipe.scheduler.calls[2][1], expected_video_noise) - torch.testing.assert_close(pipe.scheduler.calls[3][1], expected_audio_noise) - torch.testing.assert_close(pipe.scheduler.calls[2][3], video_latents - expected_video_noise) - torch.testing.assert_close(pipe.scheduler.calls[3][3], audio_latents - expected_audio_noise) video_out, audio_out = output.output - torch.testing.assert_close(video_out, (video_latents - 2 * expected_video_noise).reshape(1, 2, 1, 1, 1)) - torch.testing.assert_close(audio_out, (audio_latents - 2 * expected_audio_noise).reshape(1, 1, 1, 2)) + sigma_delta = pipe.scheduler.sigmas[-1] - pipe.scheduler.sigmas[0] + expected_video = video_latents + expected_video_noise * sigma_delta + expected_audio = audio_latents + expected_audio_noise * sigma_delta + torch.testing.assert_close(video_out, expected_video.reshape(1, 2, 1, 1, 1)) + torch.testing.assert_close(audio_out, expected_audio.reshape(1, 1, 1, 2)) # fps regression guard: an omitted request fps (frame_rate_input=None) must resolve # to the model's own 24.0 default, not crash on None; a provided rate is passed through. diff --git a/tests/diffusion/models/ltx2/test_ltx2_pipeline.py b/tests/diffusion/models/ltx2/test_ltx2_pipeline.py index 3ba36ea57..d70820f41 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_pipeline.py +++ b/tests/diffusion/models/ltx2/test_ltx2_pipeline.py @@ -6,41 +6,52 @@ import json import os import tempfile +from dataclasses import replace from types import SimpleNamespace import pytest import torch from vllm_omni.diffusion.data import DiffusionOutput +from vllm_omni.diffusion.models.ltx2 import ltx2_components from vllm_omni.diffusion.models.ltx2.ltx2_components import ( LTX2_COMPONENT_PROFILE, + LTX2_DISTILLED_COMPONENT_PROFILE, LTX23_COMPONENT_PROFILE, + _install_connector_attention, + detect_ltx_model_version, ) from vllm_omni.diffusion.models.ltx2.ltx2_conditioning import LTXI2VConditioningMixin from vllm_omni.diffusion.models.ltx2.ltx2_denoise import ( LTXDenoiseExecutor, LTXPhaseResult, + _official_ltx_sigmas, prepare_scheduler_stage, ) from vllm_omni.diffusion.models.ltx2.ltx2_guidance import ( - LTX_LEGACY_VELOCITY_GUIDANCE, - LTX_OFFICIAL_X0_GUIDANCE, + LTX_GUIDANCE_EXECUTOR, + LTXGuidancePlan, + LTXGuidanceSpec, + LTXModalityGuidance, + build_perturbation_kwargs, + combine_guided_x0, ) from vllm_omni.diffusion.models.ltx2.ltx2_latents import LTXAVState -from vllm_omni.diffusion.models.ltx2.ltx2_pipeline_runtime import LTXPipelineRuntime -from vllm_omni.diffusion.models.ltx2.ltx2_recipes import LTX2_ONE_STAGE_RECIPE, LTX23_ONE_STAGE_RECIPE -from vllm_omni.diffusion.models.ltx2.ltx2_request import LTXRequestInputs +from vllm_omni.diffusion.models.ltx2.ltx2_recipes import ( + LTX2_DISTILLED_TWO_STAGE_RECIPE, + LTX2_ONE_STAGE_RECIPE, + LTX23_ONE_STAGE_RECIPE, + LTX_DEFAULT_NEGATIVE_PROMPT, + LTX_POSITIVE_ONLY_RECIPE, +) +from vllm_omni.diffusion.models.ltx2.ltx2_request import LTXRequestInputs, validate_pipeline_request +from vllm_omni.diffusion.models.ltx2.ltx2_runtime import LTXRuntime from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import ( - LTX2ImageToVideoPipeline, + LTX2I2VDMD2Pipeline, LTX2Pipeline, - LTX23ImageToVideoPipeline, - LTX23Pipeline, - LTXOneStagePipeline, -) -from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_two_stage import ( - LTX2ImageToVideoTwoStagesPipeline, - LTX2TwoStagesPipeline, + LTX2T2VDMD2Pipeline, ) +from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_two_stage import LTX2DistilledPipeline pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -50,21 +61,28 @@ def _make_ltx_request_pipe(cls): torch.nn.Module.__init__(pipe) pipe.device = torch.device("cpu") pipe.tokenizer_max_length = 99 + pipe.vae_spatial_compression_ratio = 32 + pipe.vae_temporal_compression_ratio = 8 return pipe -def _resolve_request_inputs_for_test(pipe, req): +def _resolve_request_inputs_for_test( + pipe, + req, + *, + guidance_scale=4.0, + negative_prompt=None, +): return pipe._resolve_request_inputs( req, prompt=None, - negative_prompt=None, + negative_prompt=negative_prompt, height=None, width=None, num_frames=None, frame_rate=None, num_inference_steps=None, - timesteps=None, - guidance_scale=4.0, + guidance_scale=guidance_scale, num_videos_per_prompt=1, generator=None, latents=None, @@ -80,38 +98,69 @@ def _resolve_request_inputs_for_test(pipe, req): ) -def test_ltx_versions_share_runtime_without_cross_version_inheritance(): +def test_ltx_public_entries_share_runtime_and_keep_recipe_boundaries(): from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_ltx2 import ( DistributedAutoencoderKLLTX2Video, ) - assert issubclass(LTX2Pipeline, LTXPipelineRuntime) - assert issubclass(LTX23Pipeline, LTXPipelineRuntime) - assert issubclass(LTX2TwoStagesPipeline, LTXPipelineRuntime) - assert not issubclass(LTX23Pipeline, LTX2Pipeline) + assert issubclass(LTX2Pipeline, LTXRuntime) + assert issubclass(LTX2DistilledPipeline, LTXRuntime) assert LTX2Pipeline.component_profile is LTX2_COMPONENT_PROFILE - assert LTX23Pipeline.component_profile is LTX23_COMPONENT_PROFILE - assert LTX2Pipeline.one_stage_recipe is LTX2_ONE_STAGE_RECIPE - assert LTX23Pipeline.one_stage_recipe is LTX23_ONE_STAGE_RECIPE + assert LTX2Pipeline.pipeline_recipe is LTX2_ONE_STAGE_RECIPE + assert LTX2DistilledPipeline.component_profile is LTX2_DISTILLED_COMPONENT_PROFILE assert LTX2_COMPONENT_PROFILE.video_vae_cls is DistributedAutoencoderKLLTX2Video assert LTX23_COMPONENT_PROFILE.video_vae_cls is DistributedAutoencoderKLLTX2Video -def test_ltx_one_stage_variants_expose_shared_request_batch_and_distributed_decode_contracts(): - one_stage_variants = ( - LTX2Pipeline, - LTX2ImageToVideoPipeline, - LTX23Pipeline, - LTX23ImageToVideoPipeline, - ) - assert all(pipeline_cls.supports_request_batch for pipeline_cls in one_stage_variants) - assert all(pipeline_cls.distributed_video_decode for pipeline_cls in one_stage_variants) - assert not LTX2TwoStagesPipeline.supports_request_batch - assert not LTX2ImageToVideoTwoStagesPipeline.supports_request_batch +def test_ltx_checkpoint_version_detection_uses_metadata(tmp_path): + (tmp_path / "model_index.json").write_text(json.dumps({"vocoder": ["ltx2", "LTX2VocoderWithBWE"]})) + assert detect_ltx_model_version(str(tmp_path)) == "2.3" + + (tmp_path / "model_index.json").write_text(json.dumps({"model_version": "2.3.1"})) + assert detect_ltx_model_version(str(tmp_path)) == "2.3" + + (tmp_path / "model_index.json").write_text(json.dumps({"vocoder": ["ltx2", "LTX2Vocoder"]})) + assert detect_ltx_model_version(str(tmp_path)) == "2" -@pytest.mark.parametrize("pipeline_cls", [LTX2Pipeline, LTX23Pipeline]) -def test_ltx_request_batch_decode_splits_video_and_audio_per_request(pipeline_cls): +def test_ltx23_checkpoint_selects_version_specific_one_stage_profile(tmp_path, monkeypatch): + from vllm_omni.diffusion.models.ltx2 import ltx2_runtime + + (tmp_path / "model_index.json").write_text(json.dumps({"vocoder": ["ltx2", "LTX2VocoderWithBWE"]})) + + def stub_components(pipe, od_config): + pipe.od_config = od_config + pipe.vae_spatial_compression_ratio = 32 + + monkeypatch.setattr(ltx2_runtime, "initialize_pipeline_components", stub_components) + monkeypatch.setattr(LTXRuntime, "setup_diffusion_pipeline_profiler", lambda *_args, **_kwargs: None) + + pipe = LTX2Pipeline(od_config=SimpleNamespace(model=str(tmp_path), enable_diffusion_pipeline_profiler=False)) + + assert pipe.model_version == "2.3" + assert pipe.component_profile is LTX23_COMPONENT_PROFILE + assert pipe.pipeline_recipe is LTX23_ONE_STAGE_RECIPE + assert pipe.preserve_sp_padded_audio_duration + assert pipe.reports_stage_durations + + +def test_ltx_one_stage_entry_exposes_batch_image_and_distributed_decode_contracts(): + assert LTX2Pipeline.supports_request_batch + assert LTX2Pipeline.support_image_input + assert LTX2Pipeline.unified_text_image_entry + assert LTX2Pipeline.distributed_video_decode + assert not LTX2DistilledPipeline.supports_request_batch + assert LTX2DistilledPipeline.support_image_input + assert LTX2DistilledPipeline.unified_text_image_entry + + +def test_ltx_dmd2_entries_retain_their_task_boundaries(): + assert not LTX2T2VDMD2Pipeline.support_image_input + assert LTX2I2VDMD2Pipeline.support_image_input + assert not LTX2I2VDMD2Pipeline.unified_text_image_entry + + +def test_ltx_request_batch_decode_splits_video_and_audio_per_request(): from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch from vllm_omni.inputs.data import OmniDiffusionSamplingParams @@ -127,7 +176,7 @@ def test_ltx_request_batch_decode_splits_video_and_audio_per_request(pipeline_cl request_batch = DiffusionRequestBatch(requests) video = torch.tensor([[1.0], [2.0]]) audio = torch.tensor([[3.0], [4.0]]) - pipe = object.__new__(pipeline_cls) + pipe = object.__new__(LTX2Pipeline) torch.nn.Module.__init__(pipe) object.__setattr__(pipe, "_decode_output", lambda **_kwargs: DiffusionOutput(output=(video, audio))) forward_ctx = SimpleNamespace( @@ -159,12 +208,35 @@ def test_ltx_request_batch_decode_splits_video_and_audio_per_request(pipeline_cl torch.testing.assert_close(outputs[1].output[1], audio[1:]) -def test_ltx_one_stage_variants_share_forward_template(): - assert LTX2Pipeline._forward_impl is LTXOneStagePipeline._forward_impl - assert LTX2ImageToVideoPipeline._forward_impl is LTXOneStagePipeline._forward_impl - assert LTX23Pipeline._forward_impl is LTXOneStagePipeline._forward_impl - assert LTX23ImageToVideoPipeline._forward_impl is LTXOneStagePipeline._forward_impl - assert LTX2ImageToVideoPipeline.forward is LTX23ImageToVideoPipeline.forward +def test_ltx_public_entries_share_runtime_forward_template(): + assert "_forward_request" not in LTX2Pipeline.__dict__ + assert "forward" not in LTX2Pipeline.__dict__ + assert LTX2Pipeline._forward_request is LTXRuntime._forward_request + assert LTX2Pipeline._run_recipe is LTXRuntime._run_recipe + + +@pytest.mark.parametrize( + ("kwargs", "field_name"), + [ + ({"timesteps": [1]}, "timesteps"), + ({"guidance_rescale": 0.5}, "guidance_rescale"), + ({"noise_scale": 1.0}, "noise_scale"), + ({"return_dict": False}, "return_dict"), + ({"attention_kwargs": {}}, "attention_kwargs"), + ], +) +def test_ltx_public_forward_rejects_unsupported_generic_options(kwargs, field_name): + pipe = object.__new__(LTX2Pipeline) + + with pytest.raises(ValueError, match=field_name): + pipe.forward(SimpleNamespace(), **kwargs) + + +def test_ltx2_canonical_forward_requires_keyword_arguments(): + pipe = object.__new__(LTX2Pipeline) + + with pytest.raises(TypeError): + pipe.forward(SimpleNamespace(), "prompt") def test_ltx_versions_share_request_prompt_and_step_templates(): @@ -177,39 +249,315 @@ def test_ltx_versions_share_request_prompt_and_step_templates(): "_denoise_step", ) for method_name in shared_methods: - base_method = getattr(LTXPipelineRuntime, method_name) + base_method = getattr(LTXRuntime, method_name) assert getattr(LTX2Pipeline, method_name) is base_method - assert getattr(LTX23Pipeline, method_name) is base_method def test_ltx_versions_select_guidance_without_overriding_control_flow(): - assert LTX2Pipeline.guidance_strategy is LTX_LEGACY_VELOCITY_GUIDANCE - assert LTX23Pipeline.guidance_strategy is LTX_OFFICIAL_X0_GUIDANCE - assert LTX2Pipeline._predict_noise_for_step is LTXPipelineRuntime._predict_noise_for_step - assert LTX23Pipeline._predict_noise_for_step is LTXPipelineRuntime._predict_noise_for_step - assert LTX2Pipeline.combine_cfg_noise is LTXPipelineRuntime.combine_cfg_noise - assert LTX23Pipeline.combine_cfg_noise is LTXPipelineRuntime.combine_cfg_noise + assert LTX2Pipeline.guidance_executor is LTX_GUIDANCE_EXECUTOR + assert LTX2Pipeline._predict_noise_for_step is LTXRuntime._predict_noise_for_step + assert LTX2Pipeline.combine_cfg_noise is LTXRuntime.combine_cfg_noise -def test_ltx2_two_stage_variants_share_stage_orchestration(): - assert issubclass(LTX2ImageToVideoTwoStagesPipeline, LTX2TwoStagesPipeline) - assert issubclass(LTX2ImageToVideoTwoStagesPipeline, LTXI2VConditioningMixin) - assert LTX2ImageToVideoTwoStagesPipeline._run_two_stage is LTX2TwoStagesPipeline._run_two_stage +def test_ltx_official_recipes_build_all_guidance_passes(): + ltx2_plan = LTXGuidancePlan.build(LTX2_ONE_STAGE_RECIPE.request_guidance) + ltx23_plan = LTXGuidancePlan.build(LTX23_ONE_STAGE_RECIPE.request_guidance) + + assert ltx2_plan.names == ltx23_plan.names == ("cond", "uncond", "ptb", "mod") + assert LTX2_ONE_STAGE_RECIPE.request_guidance.video.stg_blocks == (29,) + assert LTX23_ONE_STAGE_RECIPE.request_guidance.video.stg_blocks == (28,) + assert LTX23_ONE_STAGE_RECIPE.request_guidance.video.cfg_scale == 3.0 + assert LTX23_ONE_STAGE_RECIPE.request_guidance.audio.cfg_scale == 7.0 + assert LTX2_ONE_STAGE_RECIPE.phases[0].use_official_sigma_schedule + assert LTX23_ONE_STAGE_RECIPE.phases[0].use_official_sigma_schedule + assert not LTX_POSITIVE_ONLY_RECIPE.phases[0].use_official_sigma_schedule + + +def test_ltx_guidance_combines_official_x0_deltas_in_fp32(): + cond = torch.tensor([[1.0, 3.0]], dtype=torch.bfloat16) + uncond = torch.tensor([[0.0, 1.0]], dtype=torch.bfloat16) + perturbed = torch.tensor([[0.5, 2.0]], dtype=torch.bfloat16) + isolated = torch.tensor([[0.25, 2.5]], dtype=torch.bfloat16) + guidance = LTXModalityGuidance( + cfg_scale=3.0, + stg_scale=0.5, + modality_scale=2.0, + stg_blocks=(0,), + ) + + actual = combine_guided_x0( + cond=cond, + uncond_text=uncond, + uncond_perturbed=perturbed, + uncond_modality=isolated, + guidance=guidance, + ) + expected = ( + cond.float() + + 2.0 * (cond.float() - uncond.float()) + + 0.5 * (cond.float() - perturbed.float()) + + (cond.float() - isolated.float()) + ).to(cond.dtype) + + torch.testing.assert_close(actual, expected) + + +def test_ltx_guidance_skips_stg_without_perturbed_blocks(): + cond = torch.tensor([[[1.0, 2.0], [3.0, 4.0]]]) + guidance = LTXModalityGuidance(stg_scale=1.0, stg_blocks=()) + + plan = LTXGuidancePlan.build(LTXGuidanceSpec(video=guidance)) + actual = combine_guided_x0( + cond=cond, + uncond_text=0.0, + uncond_perturbed=0.0, + uncond_modality=0.0, + guidance=guidance, + ) + + assert plan.names == ("cond",) + torch.testing.assert_close(actual, cond) + + +def test_ltx_guidance_uses_official_close_to_disabled_semantics(): + guidance = LTXModalityGuidance( + cfg_scale=1.0 + 1e-10, + stg_scale=0.0, + modality_scale=1.0 + 1e-10, + stg_blocks=(0,), + ) + + assert LTXGuidancePlan.build(LTXGuidanceSpec(video=guidance)).names == ("cond",) + + +def test_ltx_guidance_rescale_is_invariant_to_request_batching(): + cond = torch.tensor( + [ + [[1.0, 2.0], [3.0, 4.0]], + [[10.0, 30.0], [50.0, 70.0]], + ] + ) + uncond = torch.stack((torch.zeros_like(cond[0]), cond[1])) + guidance = LTXModalityGuidance(cfg_scale=2.0, rescale_scale=1.0) + + actual = combine_guided_x0( + cond=cond, + uncond_text=uncond, + uncond_perturbed=0.0, + uncond_modality=0.0, + guidance=guidance, + ) + individually = torch.cat( + [ + combine_guided_x0( + cond=cond[index : index + 1], + uncond_text=uncond[index : index + 1], + uncond_perturbed=0.0, + uncond_modality=0.0, + guidance=guidance, + ) + for index in range(cond.shape[0]) + ] + ) + + # Fusing independent requests must not change either request's output. + torch.testing.assert_close(actual, individually) + + +def test_ltx_guidance_rescale_handles_zero_variance_prediction(): + cond = torch.tensor([[[1.0, 2.0], [3.0, 4.0]]]) + guidance = LTXModalityGuidance(cfg_scale=2.0, rescale_scale=1.0) + + actual = combine_guided_x0( + cond=cond, + uncond_text=2 * cond, + uncond_perturbed=0.0, + uncond_modality=0.0, + guidance=guidance, + ) + + assert torch.isfinite(actual).all() + torch.testing.assert_close(actual, torch.zeros_like(cond)) + + +def test_ltx_guidance_builds_per_sample_stg_and_modality_masks(): + plan = LTXGuidancePlan.build(LTX23_ONE_STAGE_RECIPE.request_guidance) + perturbations = build_perturbation_kwargs(plan, batch_size=2, reference=torch.ones(8, 1, 1)) + + assert perturbations["video_self_attention_blocks"] == (28,) + assert perturbations["audio_self_attention_blocks"] == (28,) + assert perturbations["video_self_attention_mask"].flatten().tolist() == [1, 1, 1, 1, 0, 0, 1, 1] + assert perturbations["a2v_cross_attention_mask"].flatten().tolist() == [1, 1, 1, 1, 1, 1, 0, 0] + torch.testing.assert_close( + perturbations["a2v_cross_attention_mask"], + perturbations["v2a_cross_attention_mask"], + ) + + +def test_ltx_default_sigma_schedule_matches_official_shift_and_stretch(): + scheduler = SimpleNamespace( + config={ + "base_image_seq_len": 1024, + "max_image_seq_len": 4096, + "base_shift": 0.95, + "max_shift": 2.05, + "shift_terminal": 0.1, + } + ) + + sigmas = _official_ltx_sigmas(scheduler, steps=4, device=torch.device("cpu")) + + torch.testing.assert_close( + sigmas, + torch.tensor([1.0, 0.8671, 0.6316, 0.1, 0.0]), + rtol=1e-4, + atol=1e-4, + ) + + +def test_ltx_sigma_schedule_does_not_stretch_when_terminal_config_is_missing(): + scheduler = SimpleNamespace( + config={ + "base_image_seq_len": 1024, + "max_image_seq_len": 4096, + "base_shift": 0.95, + "max_shift": 2.05, + } + ) + + sigmas = _official_ltx_sigmas(scheduler, steps=4, device=torch.device("cpu")) + + torch.testing.assert_close( + sigmas, + torch.tensor([1.0, 0.9589, 0.8859, 0.7214, 0.0]), + rtol=1e-4, + atol=1e-4, + ) + + +def test_ltx_sigma_schedule_does_not_stretch_when_terminal_is_disabled(): + scheduler = SimpleNamespace( + config={ + "base_image_seq_len": 1024, + "max_image_seq_len": 4096, + "base_shift": 0.95, + "max_shift": 2.05, + "shift_terminal": None, + } + ) + + sigmas = _official_ltx_sigmas(scheduler, steps=4, device=torch.device("cpu")) + + torch.testing.assert_close( + sigmas, + torch.tensor([1.0, 0.9589, 0.8859, 0.7214, 0.0]), + rtol=1e-4, + atol=1e-4, + ) + + +def test_ltx_component_loader_installs_omni_connector_attention(monkeypatch): + installed = [] + kernels = [] + + class OmniAttention: + def __init__(self, **kwargs): + kernels.append(kwargs) + + class Attention: + heads = 32 + head_dim = 128 + inner_kv_dim = 4096 + + def set_processor(self, processor): + installed.append(processor) + + monkeypatch.setattr(ltx2_components, "OmniAttention", OmniAttention) + connectors = SimpleNamespace( + video_connector=SimpleNamespace( + learnable_registers=object(), + transformer_blocks=[SimpleNamespace(attn1=Attention())], + ), + audio_connector=SimpleNamespace( + learnable_registers=object(), + transformer_blocks=[SimpleNamespace(attn1=Attention())], + ), + ) + + _install_connector_attention(connectors) + + assert len(installed) == 2 + assert all(processor.__class__.__name__ == "_LTXConnectorAttnProcessor" for processor in installed) + assert all(processor.has_learned_registers for processor in installed) + assert all(kernel["role"] == "ltx2.connector" for kernel in kernels) + assert all(kernel["role_category"] == "self" for kernel in kernels) + assert all(kernel["skip_sequence_parallel"] for kernel in kernels) + assert all(kernel["disable_kv_quant"] for kernel in kernels) + + +@pytest.mark.parametrize("has_learned_registers", [False, True]) +def test_ltx_connector_attention_dispatches_through_omni_kernel(has_learned_registers): + calls = [] + + class Backend: + @staticmethod + def get_name(): + return "FLASH_ATTN" + + class OmniAttention: + attn_backend = Backend + + def __call__(self, query, key, value, metadata): + calls.append((query, key, value, metadata)) + return query + + attention = SimpleNamespace( + heads=2, + head_dim=4, + inner_kv_dim=8, + norm_q=torch.nn.Identity(), + norm_k=torch.nn.Identity(), + to_q=torch.nn.Identity(), + to_k=torch.nn.Identity(), + to_v=torch.nn.Identity(), + to_out=(torch.nn.Identity(), torch.nn.Identity()), + to_gate_logits=None, + rope_type="split", + omni_attention=OmniAttention(), + ) + hidden_states = torch.randn(2, 3, 8) + additive_mask = torch.zeros(2, 1, 3, 3) + + output = ltx2_components._LTXConnectorAttnProcessor(has_learned_registers=has_learned_registers)( + attention, + hidden_states, + attention_mask=additive_mask, + ) + + query, key, value, metadata = calls[0] + assert query.shape == key.shape == value.shape == (2, 3, 2, 4) + if has_learned_registers: + assert metadata is None + else: + assert metadata.attn_mask.shape == (2, 3) + assert metadata.attn_mask.dtype == torch.bool + assert metadata.attn_mask.all() + torch.testing.assert_close(output, hidden_states) + + +def test_ltx2_distilled_pipeline_shares_recipe_runtime(): + assert issubclass(LTX2DistilledPipeline, LTXI2VConditioningMixin) + assert issubclass(LTX2DistilledPipeline, LTX2Pipeline) + assert LTX2DistilledPipeline._run_recipe is LTXRuntime._run_recipe + assert "_forward_request" not in LTX2DistilledPipeline.__dict__ def test_ltx_variants_share_denoise_loop_and_i2v_conditioning(): assert "_denoise_loop" not in LTX2Pipeline.__dict__ - assert "_denoise_loop" not in LTX23Pipeline.__dict__ - assert issubclass(LTX2ImageToVideoPipeline, LTXI2VConditioningMixin) - assert issubclass(LTX23ImageToVideoPipeline, LTXI2VConditioningMixin) - assert LTX2ImageToVideoPipeline.prepare_latents is LTXI2VConditioningMixin.prepare_latents - assert LTX23ImageToVideoPipeline.prepare_latents is LTXI2VConditioningMixin.prepare_latents - assert LTX2Pipeline.prepare_latents is LTXPipelineRuntime.prepare_latents - assert LTX23Pipeline.prepare_latents is LTXPipelineRuntime.prepare_latents - assert LTX2Pipeline.prepare_audio_latents is LTXPipelineRuntime.prepare_audio_latents - assert LTX23Pipeline.prepare_audio_latents is LTXPipelineRuntime.prepare_audio_latents - assert LTX2Pipeline.decode_phase is LTXPipelineRuntime.decode_phase - assert LTX23Pipeline.decode_phase is LTXPipelineRuntime.decode_phase + assert issubclass(LTX2Pipeline, LTXI2VConditioningMixin) + assert LTX2Pipeline.prepare_latents is LTXI2VConditioningMixin.prepare_latents + assert LTX2Pipeline.prepare_audio_latents is LTXRuntime.prepare_audio_latents + assert LTX2Pipeline.decode_phase is LTXRuntime.decode_phase def test_denoise_executor_owns_progress_and_interrupt(): @@ -246,17 +594,16 @@ def test_denoise_executor_owns_progress_and_interrupt(): torch.testing.assert_close(state.audio, torch.tensor(11.0)) -def test_ltx2_two_stage_reuses_prompt_context_between_phases(): +def test_ltx2_distilled_two_stage_executes_declarative_phase_plan(): request_inputs = LTXRequestInputs( prompt="prompt", - negative_prompt="negative", - height=32, - width=32, + negative_prompt="", + height=64, + width=64, num_frames=1, frame_rate=24.0, - num_inference_steps=4, - guidance_scale=4.0, - guidance_rescale=0.0, + num_inference_steps=8, + guidance=LTXGuidanceSpec.positive_only(), num_videos_per_prompt=1, generator=None, latents=None, @@ -277,55 +624,308 @@ def test_ltx2_two_stage_reuses_prompt_context_between_phases(): return request_inputs def run_phase(req, inputs, *, prompt_context=None, **kwargs): - phase_calls.append((inputs, prompt_context)) + phase_recipe = kwargs["phase_recipe"] + assert kwargs["image"] is source_image + phase_calls.append((phase_recipe, inputs, prompt_context)) if len(phase_calls) == 1: assert prompt_context is None assert inputs.decode_timestep == 0.25 assert inputs.decode_noise_scale == 0.5 context = prompt_context_sentinel - video = torch.tensor([1.0]) - audio = torch.tensor([2.0]) + assert phase_recipe.name == "generate_lowres" + assert (inputs.height, inputs.width) == (32, 32) + assert inputs.num_inference_steps == 8 + assert kwargs["noise_scale"] == 1.0 + video = torch.ones(1, 128, 1, 1, 1) + audio = torch.full((1, 8, 1, 2), 2.0) + audio_for_next_phase = torch.full((1, 8, 1, 2), 5.0) else: assert prompt_context is prompt_context_sentinel - torch.testing.assert_close(inputs.latents, torch.tensor([11.0])) - torch.testing.assert_close(inputs.audio_latents, torch.tensor([2.0])) + assert phase_recipe.name == "refine" + assert (inputs.height, inputs.width) == (64, 64) + assert inputs.latents.shape == (1, 128, 1, 2, 2) + torch.testing.assert_close(inputs.audio_latents, torch.full((1, 8, 1, 2), 5.0)) + assert inputs.audio_latents_normalized assert inputs.guidance_scale == 1.0 assert inputs.num_inference_steps == 3 assert inputs.decode_timestep == 0.0 assert inputs.decode_noise_scale is None context = prompt_context - video = torch.tensor([3.0]) - audio = torch.tensor([4.0]) + video = torch.full((1, 128, 1, 2, 2), 3.0) + audio = torch.full((1, 8, 1, 2), 4.0) + audio_for_next_phase = None return LTXPhaseResult( forward_context=SimpleNamespace(prompt_context=context), video=video, audio=audio, + audio_for_next_phase=audio_for_next_phase, ) def decode_phase(phase): return DiffusionOutput(output=(phase.video, phase.audio)) class FakeUpsampler(torch.nn.Module): - def forward(self, *, latents, output_type, return_dict): - assert output_type == "latent" - assert not return_dict - return (latents + 10,) + dtype = torch.float32 + + def forward(self, latents): + return latents.repeat_interleave(2, dim=-2).repeat_interleave(2, dim=-1) prompt_context_sentinel = prompt_context - pipeline = object.__new__(LTX2TwoStagesPipeline) + pipeline = object.__new__(LTX2DistilledPipeline) torch.nn.Module.__init__(pipeline) - pipeline.distilled = True - pipeline.upsample_pipe = FakeUpsampler() + pipeline.device = torch.device("cpu") + pipeline.vae_spatial_compression_ratio = 32 + pipeline.vae_temporal_compression_ratio = 8 + pipeline.latent_upsampler = FakeUpsampler() object.__setattr__(pipeline, "_resolve_request_inputs", resolve_request_inputs) object.__setattr__(pipeline, "run_phase", run_phase) object.__setattr__(pipeline, "decode_phase", decode_phase) - output = pipeline.forward(SimpleNamespace()) + source_image = object() + output = pipeline.forward(SimpleNamespace(sampling_params_list=[], prompts=[]), image=source_image) assert len(phase_calls) == 2 - assert phase_calls[1][1] is prompt_context_sentinel - torch.testing.assert_close(output.output[0], torch.tensor([3.0])) - torch.testing.assert_close(output.output[1], torch.tensor([4.0])) + assert phase_calls[1][2] is prompt_context_sentinel + torch.testing.assert_close(output.output[0], torch.full((1, 128, 1, 2, 2), 3.0)) + torch.testing.assert_close(output.output[1], torch.full((1, 8, 1, 2), 4.0)) + + +def test_ltx2_distilled_stage2_reapplies_final_resolution_i2v_conditioning(): + from vllm_omni.diffusion.models.ltx2 import ltx2_latents + + pipeline = object.__new__(LTX2DistilledPipeline) + torch.nn.Module.__init__(pipeline) + pipeline.vae_spatial_compression_ratio = 32 + pipeline.vae_temporal_compression_ratio = 8 + pipeline.transformer_spatial_patch_size = 1 + pipeline.transformer_temporal_patch_size = 1 + pipeline.vae = SimpleNamespace( + latents_mean=torch.zeros(2), + latents_std=torch.ones(2), + config=SimpleNamespace(scaling_factor=1.0), + ) + object.__setattr__( + pipeline, + "_encode_i2v_image_latents", + lambda image, **_kwargs: torch.full((image.shape[0], 2, 1, 2, 2), 7.0), + ) + + packed, conditioning_mask = pipeline.prepare_latents( + image=torch.zeros(1, 3, 64, 64), + batch_size=1, + num_channels_latents=2, + height=64, + width=64, + num_frames=9, + noise_scale=0.0, + dtype=torch.float32, + device=torch.device("cpu"), + latents=torch.ones(1, 2, 2, 2, 2), + ) + unpacked = ltx2_latents.unpack_latents(packed, 2, 2, 2) + + torch.testing.assert_close(unpacked[:, :, :1], torch.full((1, 2, 1, 2, 2), 7.0)) + torch.testing.assert_close(unpacked[:, :, 1:], torch.ones(1, 2, 1, 2, 2)) + assert conditioning_mask.shape == (1, 8) + assert conditioning_mask[:, :4].all() + assert not conditioning_mask[:, 4:].any() + + +def test_ltx2_distilled_two_stage_recipe_is_fixed_positive_only(): + stage1, stage2 = LTX2_DISTILLED_TWO_STAGE_RECIPE.phases + + assert stage1.spatial_downscale == 2 + assert stage1.guidance == stage2.guidance == LTXGuidanceSpec.positive_only() + assert LTXGuidancePlan.build(stage1.guidance).names == ("cond",) + assert LTXGuidancePlan.build(stage2.guidance).names == ("cond",) + assert not stage1.allow_guidance_override + assert not stage2.allow_guidance_override + assert stage1.noise_scale == 1.0 + assert stage1.sigmas == (1.0, 0.99375, 0.9875, 0.98125, 0.975, 0.909375, 0.725, 0.421875, 0.0) + assert stage2.sigmas == (0.909375, 0.725, 0.421875, 0.0) + assert not LTX2_DISTILLED_TWO_STAGE_RECIPE.allow_request_sigmas + assert not LTX2_DISTILLED_TWO_STAGE_RECIPE.allow_request_latents + assert not LTX2_DISTILLED_TWO_STAGE_RECIPE.allow_negative_prompt + assert LTX2_DISTILLED_TWO_STAGE_RECIPE.fixed_num_inference_steps + assert (LTX2_DISTILLED_TWO_STAGE_RECIPE.height, LTX2_DISTILLED_TWO_STAGE_RECIPE.width) == (1024, 1536) + + +@pytest.mark.parametrize( + ("request_inputs", "error"), + [ + (lambda request: replace(request, height=32), "divisible by 64"), + (lambda request: replace(request, num_frames=8), "8 \\* k \\+ 1"), + (lambda request: replace(request, num_inference_steps=7), "uses 8 fixed Stage 1"), + (lambda request: replace(request, latents=torch.empty(1)), "request-provided"), + (lambda request: replace(request, audio_latents=torch.empty(1)), "request-provided"), + (lambda request: replace(request, negative_prompt="negative"), "negative conditioning"), + (lambda request: replace(request, negative_prompt_embeds=torch.empty(1)), "negative conditioning"), + (lambda request: replace(request, guidance=LTX2_ONE_STAGE_RECIPE.request_guidance), "positive-only"), + ], +) +def test_ltx_two_stage_recipe_rejects_unsupported_request_values(request_inputs, error): + request = LTXRequestInputs( + prompt="prompt", + negative_prompt="", + height=64, + width=64, + num_frames=1, + frame_rate=24.0, + num_inference_steps=8, + guidance=LTXGuidanceSpec.positive_only(), + num_videos_per_prompt=1, + generator=None, + latents=None, + audio_latents=None, + prompt_embeds=None, + negative_prompt_embeds=None, + prompt_attention_mask=None, + negative_prompt_attention_mask=None, + decode_timestep=0.0, + decode_noise_scale=None, + output_type="np", + max_sequence_length=16, + ) + + with pytest.raises(ValueError, match=error): + validate_pipeline_request( + request_inputs(request), + pipeline_recipe=LTX2_DISTILLED_TWO_STAGE_RECIPE, + vae_spatial_compression_ratio=32, + vae_temporal_compression_ratio=8, + pipeline_name="LTX2DistilledPipeline", + ) + + +def test_ltx_two_stage_recipe_rejects_sigmas(): + request = LTXRequestInputs( + prompt="prompt", + negative_prompt="", + height=64, + width=64, + num_frames=1, + frame_rate=24.0, + num_inference_steps=8, + guidance=LTXGuidanceSpec.positive_only(), + num_videos_per_prompt=1, + generator=None, + latents=None, + audio_latents=None, + prompt_embeds=None, + negative_prompt_embeds=None, + prompt_attention_mask=None, + negative_prompt_attention_mask=None, + decode_timestep=0.0, + decode_noise_scale=None, + output_type="np", + max_sequence_length=16, + ) + + with pytest.raises(ValueError, match="fixed phase sigma schedules"): + validate_pipeline_request( + request, + pipeline_recipe=LTX2_DISTILLED_TWO_STAGE_RECIPE, + vae_spatial_compression_ratio=32, + vae_temporal_compression_ratio=8, + pipeline_name="LTX2DistilledPipeline", + request_sigmas=[1.0, 0.0], + ) + + +@pytest.mark.parametrize( + ("direct_kwargs", "sampling_kwargs", "error"), + [ + ({"sigmas": [1.0, 0.0]}, {}, "fixed phase sigma schedules"), + ({}, {"sigmas": [1.0, 0.0]}, "fixed phase sigma schedules"), + ({"latents": torch.empty(1)}, {}, "request-provided video or audio latents"), + ({}, {"latents": torch.empty(1)}, "request-provided video or audio latents"), + ({"audio_latents": torch.empty(1)}, {}, "request-provided video or audio latents"), + ( + {}, + {"extra_args": {"audio_latents": torch.empty(1)}}, + "request-provided video or audio latents", + ), + ({"negative_prompt": "negative"}, {}, "negative conditioning"), + ({"negative_prompt_embeds": torch.empty(1)}, {}, "negative conditioning"), + ({}, {"num_inference_steps": 7}, "uses 8 fixed Stage 1"), + ({"guidance_scale": 4.0}, {}, "fixed positive-only guidance"), + ({}, {"guidance_scale": 4.0}, "fixed positive-only guidance"), + ], + ids=( + "direct-sigmas", + "request-sigmas", + "direct-video-latents", + "request-video-latents", + "direct-audio-latents", + "request-audio-latents", + "direct-negative-prompt", + "direct-negative-embeds", + "request-num-steps", + "direct-guidance", + "request-guidance", + ), +) +def test_ltx_distilled_forward_rejects_fixed_recipe_overrides(direct_kwargs, sampling_kwargs, error): + from vllm_omni.diffusion.request import OmniDiffusionRequest + from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch + from vllm_omni.inputs.data import OmniDiffusionSamplingParams + + sampling_values = { + "height": 64, + "width": 64, + "num_frames": 1, + "num_inference_steps": 8, + **sampling_kwargs, + } + sampling = OmniDiffusionSamplingParams(**sampling_values) + req = DiffusionRequestBatch( + [ + OmniDiffusionRequest( + prompt="prompt", + sampling_params=sampling, + request_id="ltx-distilled-fixed-inputs", + ) + ] + ) + pipeline = _make_ltx_request_pipe(LTX2DistilledPipeline) + + with pytest.raises(ValueError, match=error): + pipeline.forward(req, **direct_kwargs) + + +def test_ltx_distilled_dummy_run_uses_fixed_recipe_values(): + from vllm_omni.diffusion.request import DUMMY_DIFFUSION_REQUEST_ID, OmniDiffusionRequest + from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch + from vllm_omni.inputs.data import OmniDiffusionSamplingParams + + req = DiffusionRequestBatch( + [ + OmniDiffusionRequest( + prompt="dummy run", + sampling_params=OmniDiffusionSamplingParams( + height=512, + width=512, + num_frames=LTX2DistilledPipeline.dummy_run_num_frames, + num_inference_steps=1, + guidance_scale=0.0, + ), + request_id=DUMMY_DIFFUSION_REQUEST_ID, + ) + ] + ) + pipeline = _make_ltx_request_pipe(LTX2DistilledPipeline) + seen = {} + + def fake_run_recipe(_req, request_inputs, **_kwargs): + seen["request_inputs"] = request_inputs + return "dummy-output" + + object.__setattr__(pipeline, "_run_recipe", fake_run_recipe) + + assert pipeline.forward(req) == "dummy-output" + assert seen["request_inputs"].num_frames == 1 + assert seen["request_inputs"].num_inference_steps == 8 + assert seen["request_inputs"].guidance == LTXGuidanceSpec.positive_only() def test_ltx_custom_sigmas_bypass_scheduler_shifting(): @@ -337,10 +937,7 @@ def test_ltx_custom_sigmas_bypass_scheduler_shifting(): max_shift=2.05, shift_terminal=0.1, ) - pipeline = SimpleNamespace( - scheduler=scheduler, - _make_video_audio_scheduler=lambda *args, **kwargs: object(), - ) + pipeline = SimpleNamespace(scheduler=scheduler) sigmas = [1.0, 0.75, 0.25, 0.0] audio_scheduler, _, timesteps = prepare_scheduler_stage( @@ -352,6 +949,7 @@ def test_ltx_custom_sigmas_bypass_scheduler_shifting(): latent_num_frames=1, latent_height=1, latent_width=1, + use_official_sigma_schedule=False, ) expected = torch.tensor(sigmas, dtype=torch.float32) @@ -361,14 +959,56 @@ def test_ltx_custom_sigmas_bypass_scheduler_shifting(): class TestLTXRequestParsing: - @pytest.mark.parametrize( - ("t2v_cls", "i2v_cls"), - [ - (LTX2Pipeline, LTX2ImageToVideoPipeline), - (LTX23Pipeline, LTX23ImageToVideoPipeline), - ], - ) - def test_t2v_and_i2v_share_request_input_resolution(self, t2v_cls, i2v_cls): + def test_request_resolves_official_and_explicit_negative_prompts_per_sample(self): + from vllm_omni.diffusion.request import OmniDiffusionRequest + from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch + from vllm_omni.inputs.data import OmniDiffusionSamplingParams + + req = DiffusionRequestBatch( + [ + OmniDiffusionRequest( + prompt=prompt, + sampling_params=OmniDiffusionSamplingParams(num_inference_steps=2), + request_id=f"ltx-negative-prompt-{index}", + ) + for index, prompt in enumerate( + ( + "use the official default", + {"prompt": "disable the default", "negative_prompt": ""}, + {"prompt": "custom negative", "negative_prompt": "custom"}, + ) + ) + ] + ) + + resolved = _resolve_request_inputs_for_test(_make_ltx_request_pipe(LTX2Pipeline), req) + + assert resolved.negative_prompt == [LTX_DEFAULT_NEGATIVE_PROMPT, "", "custom"] + + def test_forward_negative_prompt_fallback_is_not_discarded(self): + from vllm_omni.diffusion.request import OmniDiffusionRequest + from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch + from vllm_omni.inputs.data import OmniDiffusionSamplingParams + + req = DiffusionRequestBatch( + [ + OmniDiffusionRequest( + prompt="prompt", + sampling_params=OmniDiffusionSamplingParams(num_inference_steps=2), + request_id="ltx-forward-negative-prompt", + ) + ] + ) + + resolved = _resolve_request_inputs_for_test( + _make_ltx_request_pipe(LTX2Pipeline), + req, + negative_prompt="forward fallback", + ) + + assert resolved.negative_prompt == ["forward fallback"] + + def test_unified_t2v_i2v_entry_resolves_request_inputs(self): from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch from vllm_omni.inputs.data import OmniDiffusionSamplingParams @@ -415,58 +1055,164 @@ class TestLTXRequestParsing: ] ) - resolved_t2v = _resolve_request_inputs_for_test( - _make_ltx_request_pipe(t2v_cls), - req, - ) - resolved_i2v = _resolve_request_inputs_for_test( - _make_ltx_request_pipe(i2v_cls), - req, - ) + resolved = _resolve_request_inputs_for_test(_make_ltx_request_pipe(LTX2Pipeline), req) - assert resolved_i2v.prompt == resolved_t2v.prompt is None - assert resolved_i2v.negative_prompt == resolved_t2v.negative_prompt is None - assert resolved_i2v.height == resolved_t2v.height == 384 - assert resolved_i2v.width == resolved_t2v.width == 512 - assert resolved_i2v.num_frames == resolved_t2v.num_frames == 25 - assert resolved_i2v.frame_rate == resolved_t2v.frame_rate == 12.5 - assert resolved_i2v.num_inference_steps == resolved_t2v.num_inference_steps == 2 - assert resolved_i2v.guidance_scale == resolved_t2v.guidance_scale == 6.0 - assert resolved_i2v.num_videos_per_prompt == resolved_t2v.num_videos_per_prompt == 2 - assert isinstance(resolved_i2v.generator, list) - assert isinstance(resolved_t2v.generator, list) - assert [gen.initial_seed() for gen in resolved_i2v.generator] == [123, 123] - assert [gen.initial_seed() for gen in resolved_t2v.generator] == [123, 123] - assert resolved_i2v.decode_timestep == resolved_t2v.decode_timestep == [0.1] - assert resolved_i2v.decode_noise_scale == resolved_t2v.decode_noise_scale == [0.2] - assert resolved_i2v.output_type == resolved_t2v.output_type == "latent" - assert resolved_i2v.max_sequence_length == resolved_t2v.max_sequence_length == 17 - torch.testing.assert_close(resolved_i2v.latents, video_latents) - torch.testing.assert_close(resolved_t2v.latents, video_latents) - torch.testing.assert_close(resolved_i2v.audio_latents, audio_latents) - torch.testing.assert_close(resolved_t2v.audio_latents, audio_latents) - torch.testing.assert_close(resolved_i2v.prompt_embeds, torch.stack([prompt_embeds])) - torch.testing.assert_close(resolved_t2v.prompt_embeds, torch.stack([prompt_embeds])) - torch.testing.assert_close(resolved_i2v.negative_prompt_embeds, torch.stack([negative_prompt_embeds])) - torch.testing.assert_close(resolved_t2v.negative_prompt_embeds, torch.stack([negative_prompt_embeds])) - torch.testing.assert_close(resolved_i2v.prompt_attention_mask, torch.stack([prompt_attention_mask])) - torch.testing.assert_close(resolved_t2v.prompt_attention_mask, torch.stack([prompt_attention_mask])) - torch.testing.assert_close( - resolved_i2v.negative_prompt_attention_mask, - torch.stack([negative_attention_mask]), - ) - torch.testing.assert_close( - resolved_t2v.negative_prompt_attention_mask, - torch.stack([negative_attention_mask]), - ) + assert resolved.prompt is None + assert resolved.negative_prompt is None + assert resolved.height == 384 + assert resolved.width == 512 + assert resolved.num_frames == 25 + assert resolved.frame_rate == 12.5 + assert resolved.num_inference_steps == 2 + assert resolved.guidance_scale == 6.0 + assert resolved.num_videos_per_prompt == 2 + assert isinstance(resolved.generator, list) + assert [gen.initial_seed() for gen in resolved.generator] == [123, 123] + assert resolved.decode_timestep == [0.1] + assert resolved.decode_noise_scale == [0.2] + assert resolved.output_type == "latent" + assert resolved.max_sequence_length == 17 + torch.testing.assert_close(resolved.latents, video_latents) + torch.testing.assert_close(resolved.audio_latents, audio_latents) + torch.testing.assert_close(resolved.prompt_embeds, torch.stack([prompt_embeds])) + torch.testing.assert_close(resolved.negative_prompt_embeds, torch.stack([negative_prompt_embeds])) + torch.testing.assert_close(resolved.prompt_attention_mask, torch.stack([prompt_attention_mask])) + torch.testing.assert_close(resolved.negative_prompt_attention_mask, torch.stack([negative_attention_mask])) - def test_request_input_resolution_rejects_mixed_precomputed_prompt_fields(self): - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23Pipeline + def test_nonbatch_distilled_request_uses_precomputed_positive_embeddings(self): from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch from vllm_omni.inputs.data import OmniDiffusionSamplingParams - pipe = _make_ltx_request_pipe(LTX23Pipeline) + prompt_embeds = torch.tensor([[1.0, 2.0]]) + prompt_attention_mask = torch.tensor([True, False]) + req = DiffusionRequestBatch( + [ + OmniDiffusionRequest( + prompt={ + "prompt": "must not be forwarded with embeddings", + "additional_information": { + "prompt_embeds": prompt_embeds, + "attention_mask": prompt_attention_mask, + }, + }, + sampling_params=OmniDiffusionSamplingParams(num_inference_steps=8), + request_id="ltx-distilled-prompt-embeds", + ) + ] + ) + + resolved = _resolve_request_inputs_for_test( + _make_ltx_request_pipe(LTX2DistilledPipeline), + req, + guidance_scale=None, + ) + + assert resolved.prompt is None + assert resolved.negative_prompt == [""] + torch.testing.assert_close(resolved.prompt_embeds, torch.stack([prompt_embeds])) + torch.testing.assert_close(resolved.prompt_attention_mask, torch.stack([prompt_attention_mask])) + + def test_request_resolves_per_modality_guidance_and_common_cfg_override(self): + from vllm_omni.diffusion.request import OmniDiffusionRequest + from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch + from vllm_omni.inputs.data import OmniDiffusionSamplingParams + + req = DiffusionRequestBatch( + [ + OmniDiffusionRequest( + prompt={"prompt": "prompt", "negative_prompt": "negative"}, + sampling_params=OmniDiffusionSamplingParams( + num_inference_steps=2, + extra_args={ + "video_cfg_scale": 4.0, + "audio_cfg_guidance_scale": 6.0, + "video_stg_guidance_scale": 0.5, + "audio_stg_scale": 0.25, + "a2v_guidance_scale": 2.0, + "v2a_guidance_scale": 4.0, + "video_rescale_scale": 0.2, + "audio_rescale_scale": 0.3, + "video_stg_blocks": [3, 4], + "audio_stg_blocks": 5, + }, + ), + request_id="ltx-guidance-overrides", + ) + ] + ) + pipe = _make_ltx_request_pipe(LTX2Pipeline) + + resolved = _resolve_request_inputs_for_test(pipe, req, guidance_scale=None) + + assert resolved.guidance.video == LTXModalityGuidance(4.0, 0.5, 2.0, 0.2, (3, 4)) + assert resolved.guidance.audio == LTXModalityGuidance(6.0, 0.25, 4.0, 0.3, (5,)) + + common = _resolve_request_inputs_for_test(pipe, req, guidance_scale=2.0) + assert common.guidance.video.cfg_scale == common.guidance.audio.cfg_scale == 2.0 + # LTX exposes independent video/audio rescale controls, not the + # model-agnostic sampling field. + assert common.guidance.video.rescale_scale == 0.2 + assert common.guidance.audio.rescale_scale == 0.3 + + def test_request_rejects_unsupported_generic_scheduler_and_rescale_controls(self): + from vllm_omni.diffusion.request import OmniDiffusionRequest + from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch + from vllm_omni.inputs.data import OmniDiffusionSamplingParams + + cases = ( + (OmniDiffusionSamplingParams(timesteps=torch.tensor([1.0])), "timesteps"), + (OmniDiffusionSamplingParams(guidance_rescale=0.5), "guidance_rescale"), + (OmniDiffusionSamplingParams(extra_args={"flow_shift": 3.0}), "flow_shift"), + ) + pipe = _make_ltx_request_pipe(LTX2Pipeline) + + for index, (sampling, field_name) in enumerate(cases): + req = DiffusionRequestBatch( + [ + OmniDiffusionRequest( + prompt="prompt", + sampling_params=sampling, + request_id=f"ltx-unsupported-{index}", + ) + ] + ) + with pytest.raises(ValueError, match=field_name): + _resolve_request_inputs_for_test(pipe, req) + + def test_request_batch_rejects_mixed_guidance_specs(self): + from vllm_omni.diffusion.request import OmniDiffusionRequest + from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch + from vllm_omni.inputs.data import OmniDiffusionSamplingParams + + req = DiffusionRequestBatch( + [ + OmniDiffusionRequest( + prompt="prompt-0", + sampling_params=OmniDiffusionSamplingParams( + num_inference_steps=2, + extra_args={"video_cfg_scale": scale}, + ), + request_id=f"ltx-mixed-guidance-{index}", + ) + for index, scale in enumerate((3.0, 4.0)) + ] + ) + + with pytest.raises(ValueError, match="identical video/audio guidance"): + _resolve_request_inputs_for_test( + _make_ltx_request_pipe(LTX2Pipeline), + req, + guidance_scale=None, + ) + + def test_request_input_resolution_rejects_mixed_precomputed_prompt_fields(self): + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline + from vllm_omni.diffusion.request import OmniDiffusionRequest + from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch + from vllm_omni.inputs.data import OmniDiffusionSamplingParams + + pipe = _make_ltx_request_pipe(LTX2Pipeline) req = DiffusionRequestBatch( [ OmniDiffusionRequest( @@ -499,11 +1245,40 @@ class TestLTXRequestParsing: _resolve_request_inputs_for_test(pipe, req) -class TestLTX23ForwardStages: - def test_encode_prompt_repeats_precomputed_embeds_for_num_outputs(self): - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23Pipeline +class TestLTXForwardStages: + def test_encode_prompt_batches_positive_and_negative_gemma_inputs(self): + pipe = _make_ltx_request_pipe(LTX2Pipeline) + calls = [] - pipe = _make_ltx_request_pipe(LTX23Pipeline) + def fake_get_gemma_prompt_embeds(**kwargs): + calls.append(kwargs) + count = len(kwargs["prompt"]) + embeds = torch.arange(count, dtype=torch.float32).view(count, 1, 1) + mask = torch.arange(count).view(count, 1) + return embeds, mask + + object.__setattr__(pipe, "_get_gemma_prompt_embeds", fake_get_gemma_prompt_embeds) + + prompt_embeds, prompt_mask, negative_embeds, negative_mask = pipe.encode_prompt( + prompt=["positive-0", "positive-1"], + negative_prompt=["negative-0", "negative-1"], + do_classifier_free_guidance=True, + num_videos_per_prompt=2, + device=torch.device("cpu"), + ) + + assert len(calls) == 1 + assert calls[0]["prompt"] == ["positive-0", "positive-1", "negative-0", "negative-1"] + assert calls[0]["num_videos_per_prompt"] == 1 + torch.testing.assert_close(prompt_embeds.flatten(), torch.tensor([0.0, 0.0, 1.0, 1.0])) + torch.testing.assert_close(negative_embeds.flatten(), torch.tensor([2.0, 2.0, 3.0, 3.0])) + torch.testing.assert_close(prompt_mask.flatten(), torch.tensor([0, 0, 1, 1])) + torch.testing.assert_close(negative_mask.flatten(), torch.tensor([2, 2, 3, 3])) + + def test_encode_prompt_repeats_precomputed_embeds_for_num_outputs(self): + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline + + pipe = _make_ltx_request_pipe(LTX2Pipeline) prompt_embeds = torch.tensor([[[1.0], [2.0]], [[3.0], [4.0]]]) negative_prompt_embeds = torch.tensor([[[-1.0], [-2.0]], [[-3.0], [-4.0]]]) prompt_attention_mask = torch.tensor([[True, False], [False, True]]) @@ -542,13 +1317,13 @@ class TestLTX23ForwardStages: torch.tensor([[False, False], [False, False], [True, True], [True, True]]), ) - def test_t2v_forward_delegates_to_shared_forward_impl(self): - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23Pipeline + def test_t2v_forward_delegates_to_shared_recipe_runtime(self): + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch from vllm_omni.inputs.data import OmniDiffusionSamplingParams - pipe = _make_ltx_request_pipe(LTX23Pipeline) + pipe = _make_ltx_request_pipe(LTX2Pipeline) request_sigmas = [1.0, 0.4] req = DiffusionRequestBatch( [ @@ -569,25 +1344,17 @@ class TestLTX23ForwardStages: ] ) fallback_sigmas = [1.0, 0.5] - timesteps = [1000, 500] - attention_kwargs = {"scale": 1.0} seen = {} - def fake_forward_impl(req_arg, request_inputs, **kwargs): + def fake_run_recipe(req_arg, request_inputs, **kwargs): seen["req"] = req_arg seen["request_inputs"] = request_inputs seen["kwargs"] = kwargs return ["delegated"] - object.__setattr__(pipe, "_forward_impl", fake_forward_impl) + object.__setattr__(pipe, "_run_recipe", fake_run_recipe) - output = pipe.forward( - req, - sigmas=fallback_sigmas, - timesteps=timesteps, - noise_scale=0.25, - attention_kwargs=attention_kwargs, - ) + output = pipe.forward(req, sigmas=fallback_sigmas) assert output == ["delegated"] assert seen["req"] is req @@ -598,20 +1365,52 @@ class TestLTX23ForwardStages: assert seen["request_inputs"].guidance_scale == 4.5 assert seen["request_inputs"].output_type == "latent" assert seen["kwargs"] == { - "noise_scale": 0.25, - "sigmas": request_sigmas, - "timesteps": timesteps, - "attention_kwargs": attention_kwargs, + "request_sigmas": request_sigmas, + "image": None, } + def test_t2v_forward_prefers_request_sigmas(self): + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline + from vllm_omni.diffusion.request import OmniDiffusionRequest + from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch + from vllm_omni.inputs.data import OmniDiffusionSamplingParams + + request_sigmas = [1.0, 0.25] + req = DiffusionRequestBatch( + [ + OmniDiffusionRequest( + prompt="custom sigma schedule", + sampling_params=OmniDiffusionSamplingParams( + num_inference_steps=2, + sigmas=request_sigmas, + ), + request_id="ltx23-request-sigmas", + ) + ] + ) + pipe = _make_ltx_request_pipe(LTX2Pipeline) + seen = {} + + def fake_run_recipe(req_arg, request_inputs, **kwargs): + del req_arg, request_inputs + seen.update(kwargs) + return ["delegated"] + + object.__setattr__(pipe, "_run_recipe", fake_run_recipe) + + output = pipe.forward(req, sigmas=[1.0, 0.5]) + + assert output == ["delegated"] + assert seen["request_sigmas"] == request_sigmas + class TestPipelineComponents: - def test_ltx23_pipeline_declares_offload_components(self): - """LTX23Pipeline must expose LTX-2.3-specific modules to offload discovery.""" - from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX23Pipeline + def test_ltx_pipeline_declares_offload_components(self): + """LTX2Pipeline exposes all one-stage modules to offload discovery.""" + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline from vllm_omni.diffusion.offloader.module_collector import ModuleDiscovery - pipe = object.__new__(LTX23Pipeline) + pipe = object.__new__(LTX2Pipeline) torch.nn.Module.__init__(pipe) pipe.transformer = torch.nn.Linear(1, 1) pipe.text_encoder = torch.nn.Linear(1, 1) @@ -622,19 +1421,22 @@ class TestPipelineComponents: modules = ModuleDiscovery.discover(pipe) - assert LTX23Pipeline._dit_modules == ["transformer"] - assert LTX23Pipeline._encoder_modules == ["text_encoder", "connectors"] - assert LTX23Pipeline._vae_modules == ["vae", "audio_vae"] - assert LTX23Pipeline._resident_modules == ["vocoder"] + assert LTX2Pipeline._dit_modules == ["transformer"] + assert LTX2Pipeline._encoder_modules == ["text_encoder", "connectors"] + assert LTX2Pipeline._vae_modules == ["vae", "audio_vae"] + assert LTX2Pipeline._resident_modules == ["vocoder"] assert modules.dit_names == ["transformer"] assert modules.encoder_names == ["text_encoder", "connectors"] assert modules.resident_names == ["vocoder"] assert len(modules.vaes) == 2 + def test_distilled_profile_keeps_upsampler_in_managed_resident_components(self): + assert LTX2_DISTILLED_COMPONENT_PROFILE.resident_modules == ("vocoder", "latent_upsampler") + class TestLTX23DecodeConditioning: def test_decode_conditioning_expands_per_prompt_values_to_effective_batch(self): - from vllm_omni.diffusion.models.ltx2.ltx2_pipeline_runtime import _expand_per_prompt_decode_value + from vllm_omni.diffusion.models.ltx2.ltx2_runtime import _expand_per_prompt_decode_value assert _expand_per_prompt_decode_value( [0.1, 0.2], @@ -656,7 +1458,7 @@ class TestLTX23DecodeConditioning: ) == [0.1, 0.2, 0.3, 0.4] def test_decode_conditioning_rejects_ambiguous_lengths(self): - from vllm_omni.diffusion.models.ltx2.ltx2_pipeline_runtime import _expand_per_prompt_decode_value + from vllm_omni.diffusion.models.ltx2.ltx2_runtime import _expand_per_prompt_decode_value with pytest.raises(ValueError, match="decode_timestep"): _expand_per_prompt_decode_value( @@ -668,19 +1470,26 @@ class TestLTX23DecodeConditioning: class TestRegistryIntegration: - """Verify all LTX-2.3 pipeline variants are registered.""" + """Verify the unified one-stage and existing two-stage entries.""" def test_pipeline_module_paths(self): """Registry entries must point to the correct modules.""" from vllm_omni.diffusion.registry import _DIFFUSION_MODELS - assert _DIFFUSION_MODELS["LTX23Pipeline"] == ("ltx2", "pipeline_ltx2", "LTX23Pipeline") - - assert _DIFFUSION_MODELS["LTX23ImageToVideoPipeline"] == ( + assert _DIFFUSION_MODELS["LTX2Pipeline"] == ("ltx2", "pipeline_ltx2", "LTX2Pipeline") + assert _DIFFUSION_MODELS["LTX2DistilledPipeline"] == ( "ltx2", - "pipeline_ltx2", - "LTX23ImageToVideoPipeline", + "pipeline_ltx2_two_stage", + "LTX2DistilledPipeline", ) + removed_entries = { + "LTX2ImageToVideoPipeline", + "LTX23Pipeline", + "LTX23ImageToVideoPipeline", + "LTX2TwoStagesPipeline", + "LTX2ImageToVideoTwoStagesPipeline", + } + assert removed_entries.isdisjoint(_DIFFUSION_MODELS) def test_post_process_funcs_registered(self): """Pipeline variants must map to get_ltx2_post_process_func.""" @@ -688,13 +1497,9 @@ class TestRegistryIntegration: expected = [ "LTX2Pipeline", - "LTX2ImageToVideoPipeline", - "LTX2TwoStagesPipeline", - "LTX2ImageToVideoTwoStagesPipeline", + "LTX2DistilledPipeline", "LTX2T2VDMD2Pipeline", "LTX2I2VDMD2Pipeline", - "LTX23Pipeline", - "LTX23ImageToVideoPipeline", ] for name in expected: assert name in _DIFFUSION_POST_PROCESS_FUNCS, f"{name} not in _DIFFUSION_POST_PROCESS_FUNCS" @@ -704,13 +1509,9 @@ class TestRegistryIntegration: "model_class_name", [ "LTX2Pipeline", - "LTX2ImageToVideoPipeline", - "LTX2TwoStagesPipeline", - "LTX2ImageToVideoTwoStagesPipeline", + "LTX2DistilledPipeline", "LTX2T2VDMD2Pipeline", "LTX2I2VDMD2Pipeline", - "LTX23Pipeline", - "LTX23ImageToVideoPipeline", ], ) def test_post_process_func_resolves_from_every_entry_module(self, model_class_name): @@ -735,10 +1536,7 @@ class TestRegistryIntegration: # NOTE: We used to have custom enablers for this model, but refactored to handle # it more generically. Now we only need to ensure it has git cache adapter config. - expected = [ - "LTX23Pipeline", - "LTX23ImageToVideoPipeline", - ] + expected = ["LTX2Pipeline"] for name in expected: assert name not in CUSTOM_DIT_ENABLERS, f"{name} not in CUSTOM_DIT_ENABLERS" diff --git a/tests/diffusion/models/ltx2/test_ltx2_transformer.py b/tests/diffusion/models/ltx2/test_ltx2_transformer.py index e5ccfd013..de037fef8 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_transformer.py +++ b/tests/diffusion/models/ltx2/test_ltx2_transformer.py @@ -10,11 +10,12 @@ from vllm_omni.diffusion.models.ltx2.ltx2_transformer import LTX2VideoTransforme pytestmark = [pytest.mark.core_model, pytest.mark.cpu] -def test_ltx_rms_norm_no_affine_identity_weight_is_non_persistent_buffer(): +def test_ltx_rms_norm_no_affine_matches_official_torch_module(): norm = _make_rms_norm(8, eps=1e-6, elementwise_affine=False) assert "weight" not in dict(norm.named_parameters()) - assert "weight" in dict(norm.named_buffers()) + assert norm.weight is None + assert "weight" not in dict(norm.named_buffers()) assert "weight" not in norm.state_dict() @@ -50,3 +51,16 @@ def test_ltx_transformer_exposes_hsdp_shard_conditions_for_blocks(): matched.append(name) assert matched == ["transformer_blocks.0", "transformer_blocks.1"] + + +@pytest.mark.parametrize("rope_type", ["interleaved", "split"]) +def test_ltx_sp_plan_shards_video_and_audio_timesteps_together(rope_type): + root_plan = LTX2VideoTransformer3DModel._build_sp_plan(rope_type)[""] + + video_timestep = root_plan["timestep"] + audio_timestep = root_plan["audio_timestep"] + + assert video_timestep.split_dim == audio_timestep.split_dim == 1 + assert video_timestep.expected_dims == audio_timestep.expected_dims == 2 + assert not video_timestep.split_output + assert not audio_timestep.split_output diff --git a/tests/diffusion/models/ltx2/test_ltx2_vae.py b/tests/diffusion/models/ltx2/test_ltx2_vae.py index 4ae7a0721..108bb960d 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_vae.py +++ b/tests/diffusion/models/ltx2/test_ltx2_vae.py @@ -14,7 +14,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu] class TestLTXOutputRank: @pytest.mark.parametrize( ("distributed_vae_state", "expected_decode_calls"), - [(False, 0), (True, 1), (RuntimeError("unavailable"), 0)], + [(False, 0), (True, 1), (RuntimeError("unavailable"), None)], ) def test_non_output_rank_only_enters_collective_vae_decode( self, @@ -22,7 +22,8 @@ class TestLTXOutputRank: distributed_vae_state, expected_decode_calls, ): - import vllm_omni.diffusion.models.ltx2.pipeline_ltx2 as ltx + import vllm_omni.diffusion.models.ltx2.ltx2_runtime as ltx_runtime + from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline class FakeVae: dtype = torch.float32 @@ -40,23 +41,30 @@ class TestLTXOutputRank: self.decode_calls += 1 return (torch.ones(1, 1),) - monkeypatch.setattr(ltx.torch.distributed, "is_initialized", lambda: True) - monkeypatch.setattr(ltx.torch.distributed, "get_rank", lambda: 1) - pipe = object.__new__(ltx.LTX2Pipeline) + monkeypatch.setattr(ltx_runtime.torch.distributed, "is_initialized", lambda: True) + monkeypatch.setattr(ltx_runtime.torch.distributed, "get_rank", lambda: 1) + pipe = object.__new__(LTX2Pipeline) torch.nn.Module.__init__(pipe) pipe.vae = FakeVae() - output = pipe._decode_output( - latents=torch.ones(1, 1), - audio_latents=torch.ones(1, 1), - output_type="np", - connector_prompt_embeds=torch.ones(1, 1), - generator=None, - device=torch.device("cpu"), - decode_timestep=0.0, - decode_noise_scale=None, - prompt_batch_size=1, - ) + decode_kwargs = { + "latents": torch.ones(1, 1), + "audio_latents": torch.ones(1, 1), + "output_type": "np", + "connector_prompt_embeds": torch.ones(1, 1), + "generator": None, + "device": torch.device("cpu"), + "decode_timestep": 0.0, + "decode_noise_scale": None, + "prompt_batch_size": 1, + } + if isinstance(distributed_vae_state, Exception): + with pytest.raises(RuntimeError, match="unavailable"): + pipe._decode_output(**decode_kwargs) + assert pipe.vae.decode_calls == 0 + return + + output = pipe._decode_output(**decode_kwargs) assert pipe.vae.decode_calls == expected_decode_calls assert output.output[0].numel() == 0 diff --git a/tests/e2e/accuracy/ltx/__init__.py b/tests/e2e/accuracy/ltx/__init__.py new file mode 100644 index 000000000..08143263b --- /dev/null +++ b/tests/e2e/accuracy/ltx/__init__.py @@ -0,0 +1 @@ +"""LTX family official-reference accuracy tests.""" diff --git a/tests/e2e/accuracy/ltx/run_ltx_reference.py b/tests/e2e/accuracy/ltx/run_ltx_reference.py new file mode 100644 index 000000000..a4a180047 --- /dev/null +++ b/tests/e2e/accuracy/ltx/run_ltx_reference.py @@ -0,0 +1,331 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Generate raw LTX video and audio outputs with the official or Omni runtime.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +import numpy as np +import torch +from PIL import Image + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--backend", choices=("official", "omni"), required=True) + parser.add_argument("--request", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--model") + parser.add_argument("--model-class-name") + parser.add_argument("--official-root", type=Path) + parser.add_argument("--checkpoint", type=Path) + parser.add_argument("--gemma-root", type=Path) + parser.add_argument("--enable-layerwise-offload", action="store_true") + return parser.parse_args() + + +def _save_outputs( + output_dir: Path, + *, + video: torch.Tensor, + audio: torch.Tensor, + audio_sample_rate: int, + metadata: dict[str, Any], +) -> None: + output_dir.mkdir(parents=True, exist_ok=True) + video = video.detach().float().cpu().clamp(0.0, 1.0) + audio = audio.detach().float().cpu() + np.save(output_dir / "video.npy", video.numpy()) + np.save(output_dir / "audio.npy", audio.numpy()) + + frame_indices = sorted({0, video.shape[0] // 2, video.shape[0] - 1}) + for index in frame_indices: + frame = video[index].mul(255.0).round().to(torch.uint8).numpy() + Image.fromarray(frame).save(output_dir / f"frame_{index:04d}.png") + + metadata.update( + { + "video_shape": list(video.shape), + "audio_shape": list(audio.shape), + "audio_sample_rate": int(audio_sample_rate), + "frame_indices": frame_indices, + } + ) + (output_dir / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n") + + +def _insert_official_paths(official_root: Path) -> None: + for relative_path in ("packages/ltx-core/src", "packages/ltx-pipelines/src"): + path = str((official_root / relative_path).resolve()) + if path not in sys.path: + sys.path.insert(0, path) + + +def _configure_official_sdpa(pipeline: Any) -> None: + """Use PyTorch SDPA for official connector and denoiser attention.""" + from ltx_core.loader.attention_ops import set_attention_module_op + from ltx_core.model.transformer.attention import PytorchAttention + + class AllValidSDPA(PytorchAttention): + def __call__( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + heads: int, + mask: torch.Tensor | None = None, + ) -> torch.Tensor: + if mask is not None and torch.count_nonzero(mask).item(): + raise ValueError("The LTX accuracy guard cannot discard a non-empty attention mask") + return super().__call__(query, key, value, heads, mask=None) + + attention = AllValidSDPA() + module_op = set_attention_module_op( + attention=attention, + masked_attention=attention, + ) + owners_and_attributes = ( + (pipeline.stage, "_transformer_builder"), + (pipeline.prompt_encoder, "_embeddings_processor_builder"), + ) + for owner, attribute in owners_and_attributes: + builder = getattr(owner, attribute) + setattr( + owner, + attribute, + builder.with_module_ops((*builder.module_ops, module_op)), + ) + + +@torch.inference_mode() +def _run_official(args: argparse.Namespace, request: dict[str, Any]) -> None: + if args.official_root is None or args.checkpoint is None or args.gemma_root is None: + raise ValueError("Official backend requires --official-root, --checkpoint, and --gemma-root") + _insert_official_paths(args.official_root) + # vLLM disables cuDNN SDPA during import; mirror the worker's Gemma dispatch. + torch.backends.cuda.enable_cudnn_sdp(False) + + from ltx_core.components.guiders import MultiModalGuiderParams + from ltx_pipelines.ti2vid_one_stage import TI2VidOneStagePipeline + from ltx_pipelines.utils.args import ImageConditioningInput + from ltx_pipelines.utils.types import OffloadMode + + pipeline = TI2VidOneStagePipeline( + checkpoint_path=str(args.checkpoint), + gemma_root=str(args.gemma_root), + loras=(), + offload_mode=OffloadMode.CPU if args.enable_layerwise_offload else OffloadMode.NONE, + ) + _configure_official_sdpa(pipeline) + image_path = request.get("image") + images = ( + [] + if image_path is None + else [ + ImageConditioningInput( + path=str(image_path), + frame_idx=0, + strength=1.0, + # Both runtimes must receive the same source pixels. The + # official CLI's optional H.264 preprocessing is not part of + # the seeded model trajectory under test. + crf=0, + ) + ] + ) + video, audio = pipeline( + prompt=request["prompt"], + negative_prompt=request["negative_prompt"], + seed=request["seed"], + height=request["height"], + width=request["width"], + num_frames=request["num_frames"], + frame_rate=request["fps"], + num_inference_steps=request["num_inference_steps"], + video_guider_params=MultiModalGuiderParams( + cfg_scale=request["video_cfg_scale"], + stg_scale=request["video_stg_scale"], + rescale_scale=request["video_rescale_scale"], + modality_scale=request["video_modality_scale"], + skip_step=0, + stg_blocks=request["video_stg_blocks"], + ), + audio_guider_params=MultiModalGuiderParams( + cfg_scale=request["audio_cfg_scale"], + stg_scale=request["audio_stg_scale"], + rescale_scale=request["audio_rescale_scale"], + modality_scale=request["audio_modality_scale"], + skip_step=0, + stg_blocks=request["audio_stg_blocks"], + ), + images=images, + max_batch_size=4, + ) + video_tensor = torch.cat([chunk.detach().cpu() for chunk in video], dim=0) + _save_outputs( + args.output_dir, + video=_canonical_video(video_tensor), + audio=audio.waveform, + audio_sample_rate=audio.sampling_rate, + metadata={ + "backend": "official", + "attention_backend": "torch_sdpa", + "official_revision": os.environ.get("VLLM_TEST_LTX_OFFICIAL_REVISION"), + "checkpoint": str(args.checkpoint), + }, + ) + + +def _unwrap_omni_output(output: Any) -> tuple[Any, Any, int]: + from vllm_omni.outputs import OmniRequestOutput + + audio = None + audio_sample_rate = None + frames = output[0] if isinstance(output, list) and output else output + if isinstance(frames, OmniRequestOutput): + multimodal_output = frames.multimodal_output or {} + audio = multimodal_output.get("audio") + audio_sample_rate = multimodal_output.get("audio_sample_rate") + if frames.is_pipeline_output and isinstance(frames.request_output, OmniRequestOutput): + frames = frames.request_output + multimodal_output = frames.multimodal_output or {} + audio = multimodal_output.get("audio", audio) + audio_sample_rate = multimodal_output.get("audio_sample_rate", audio_sample_rate) + if isinstance(frames, OmniRequestOutput): + if not frames.images: + raise ValueError("No video frames found in OmniRequestOutput") + frames = frames.images + + if isinstance(frames, list) and len(frames) == 1: + frames = frames[0] + if isinstance(frames, tuple) and len(frames) == 2: + frames, audio = frames + elif isinstance(frames, dict): + audio = frames.get("audio", audio) + audio_sample_rate = frames.get("audio_sample_rate", audio_sample_rate) + frames = frames.get("frames", frames.get("video")) + + if frames is None or audio is None or audio_sample_rate is None: + raise ValueError("Omni output did not contain video, audio, and audio_sample_rate") + return frames, audio, int(audio_sample_rate) + + +def _canonical_video(video: Any) -> torch.Tensor: + if isinstance(video, list): + if len(video) == 1: + return _canonical_video(video[0]) + tensors = [torch.as_tensor(np.asarray(item) if not isinstance(item, torch.Tensor) else item) for item in video] + if all(item.ndim == 3 for item in tensors): + video = torch.stack(tensors) + elif all(item.ndim == 4 for item in tensors): + video = torch.cat(tensors) + else: + raise ValueError(f"Cannot combine video items with dimensions {[item.ndim for item in tensors]}") + tensor = torch.as_tensor(np.asarray(video) if not isinstance(video, torch.Tensor) else video).detach().cpu() + if tensor.ndim == 5: + tensor = tensor[0] + if tensor.ndim != 4: + raise ValueError(f"Expected a 4D video tensor, got {tuple(tensor.shape)}") + if tensor.shape[-1] in (3, 4): + tensor = tensor[..., :3] + elif tensor.shape[1] in (3, 4): + tensor = tensor[:, :3].permute(0, 2, 3, 1) + elif tensor.shape[0] in (3, 4): + tensor = tensor[:3].permute(1, 2, 3, 0) + else: + raise ValueError(f"Cannot infer video channel dimension from {tuple(tensor.shape)}") + tensor = tensor.float() + if tensor.numel() and tensor.max() > 1: + tensor = tensor / 255.0 + elif tensor.numel() and tensor.min() < 0: + tensor = tensor.clamp(-1.0, 1.0).add(1.0).mul(0.5) + return tensor.clamp(0.0, 1.0) + + +@torch.inference_mode() +def _run_omni(args: argparse.Namespace, request: dict[str, Any]) -> None: + if args.model is None or args.model_class_name is None: + raise ValueError("Omni backend requires --model and --model-class-name") + + attention_config = {"default": {"backend": "TORCH_SDPA"}} + + from vllm_omni.diffusion.data import DiffusionParallelConfig + from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args + from vllm_omni.entrypoints.omni import Omni + from vllm_omni.inputs.data import OmniDiffusionSamplingParams + from vllm_omni.model_extras import get_extra_body_params, get_model_class_name + from vllm_omni.platforms import current_omni_platform + + generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(request["seed"]) + omni = Omni( + model=args.model, + model_class_name=args.model_class_name, + enforce_eager=True, + enable_layerwise_offload=args.enable_layerwise_offload, + diffusion_attention_config=attention_config, + parallel_config=DiffusionParallelConfig(), + ) + try: + model_class_name = get_model_class_name(omni) + sampling_params = OmniDiffusionSamplingParams( + height=request["height"], + width=request["width"], + generator=generator, + guidance_scale=None, + num_inference_steps=request["num_inference_steps"], + num_frames=request["num_frames"], + fps=request["fps"], + frame_rate=float(request["fps"]), + output_type="np", + ) + guidance = { + key: value for key, value in request.items() if key.startswith("video_") or key.startswith("audio_") + } + apply_declared_extra_args(sampling_params, get_extra_body_params(model_class_name), guidance) + prompt: dict[str, Any] = { + "prompt": request["prompt"], + "negative_prompt": request["negative_prompt"], + } + image_path = request.get("image") + if image_path is not None: + with Image.open(str(image_path)) as source_image: + image = source_image.convert("RGB") + image.load() + prompt["multi_modal_data"] = {"image": image} + output = omni.generate(prompt, sampling_params) + video, audio, audio_sample_rate = _unwrap_omni_output(output) + audio_tensor = torch.as_tensor(np.asarray(audio) if not isinstance(audio, torch.Tensor) else audio) + _save_outputs( + args.output_dir, + video=_canonical_video(video), + audio=audio_tensor, + audio_sample_rate=audio_sample_rate, + metadata={ + "backend": "omni", + "attention_backend": "torch_sdpa", + "model": args.model, + "model_class_name": model_class_name, + }, + ) + finally: + omni.shutdown() + + +def main() -> None: + args = _parse_args() + request = json.loads(args.request.read_text()) + if args.backend == "official": + _run_official(args, request) + else: + _run_omni(args, request) + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/accuracy/ltx/test_ltx_official_similarity.py b/tests/e2e/accuracy/ltx/test_ltx_official_similarity.py new file mode 100644 index 000000000..647ece7a7 --- /dev/null +++ b/tests/e2e/accuracy/ltx/test_ltx_official_similarity.py @@ -0,0 +1,440 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""E2E accuracy guard against a pinned Lightricks LTX pipeline revision. + +The comparison runs both runtimes through PyTorch SDPA and uses +``max_batch_size=4`` in the official reference to match Omni's fused guidance +batch. Video and audio guidance use the official non-HQ one-stage defaults; +only the generation shape and step count are reduced for CI runtime. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pytest +import torch +from huggingface_hub import hf_hub_download, snapshot_download +from torchmetrics.image import PeakSignalNoiseRatio, StructuralSimilarityIndexMeasure + +from tests.e2e.accuracy.helpers import reset_artifact_dir +from tests.helpers.mark import hardware_test + +OFFICIAL_REPOSITORY = "https://github.com/Lightricks/LTX-2.git" +OFFICIAL_REVISION = "9377758131b1ffde4b7f766804590a6617bf2ab9" +# Version selected by this revision's uv.lock. Keep it out of Omni's runtime and dev dependencies. +OFFICIAL_OPENIMAGEIO_VERSION = "3.1.11.0" +PROMPT = ( + "A space shuttle launches vertically above a desert launch pad. Bright exhaust flames and a dense white " + "plume billow beneath it while the camera remains fixed." +) +NEGATIVE_PROMPT = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field, " + "background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent lighting " + "direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny valley effect, " + "incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, mismatched lip sync, " + "silent or muted audio, distorted voice, robotic voice, echo, background noise, off-sync audio, incorrect " + "dialogue, added dialogue, repetitive speech, jittery movement, awkward pauses, incorrect timing, unnatural " + "transitions, inconsistent framing, tilted camera, flat lighting, inconsistent tone, cinematic oversaturation, " + "stylized filters, or AI artifacts." +) + +# Both runtimes use PyTorch SDPA with the current Torch dispatch defaults. +ATTENTION_BACKEND = "torch_sdpa" +VIDEO_SSIM_MEAN_THRESHOLD = 0.95 +VIDEO_SSIM_MIN_THRESHOLD = 0.90 +VIDEO_PSNR_MEAN_THRESHOLD = 30.0 +AUDIO_RELATIVE_L2_THRESHOLD = 0.2 +AUDIO_COSINE_THRESHOLD = 0.95 + + +@dataclass(frozen=True) +class LTXAccuracyCase: + name: str + model_id: str + model_revision: str + model_env: str + model_class_name: str + checkpoint_repo: str + checkpoint_filename: str + checkpoint_revision: str + checkpoint_env: str + stg_block: int + prompt: str = PROMPT + image_repo: str | None = None + image_filename: str | None = None + image_revision: str | None = None + + +CASES = ( + LTXAccuracyCase( + name="ltx2", + model_id="Lightricks/LTX-2", + model_revision="47da56e2ad66ce4125a9922b4a8826bf407f9d0a", + model_env="VLLM_TEST_LTX2_MODEL", + model_class_name="LTX2Pipeline", + checkpoint_repo="Lightricks/LTX-2", + checkpoint_filename="ltx-2-19b-dev.safetensors", + checkpoint_revision="47da56e2ad66ce4125a9922b4a8826bf407f9d0a", + checkpoint_env="VLLM_TEST_LTX2_OFFICIAL_CHECKPOINT", + stg_block=29, + ), + LTXAccuracyCase( + name="ltx2_3", + model_id="diffusers/LTX-2.3-Diffusers", + model_revision="8eee8edcf067e838b843f926ec4d4cc9b2be1aaf", + model_env="VLLM_TEST_LTX23_MODEL", + model_class_name="LTX2Pipeline", + checkpoint_repo="Lightricks/LTX-2.3", + checkpoint_filename="ltx-2.3-22b-dev.safetensors", + checkpoint_revision="4229404625088d21c4f112eb640fb04a0900ee25", + checkpoint_env="VLLM_TEST_LTX23_OFFICIAL_CHECKPOINT", + stg_block=28, + ), + LTXAccuracyCase( + name="ltx2_3_i2v", + model_id="diffusers/LTX-2.3-Diffusers", + model_revision="8eee8edcf067e838b843f926ec4d4cc9b2be1aaf", + model_env="VLLM_TEST_LTX23_MODEL", + model_class_name="LTX2Pipeline", + checkpoint_repo="Lightricks/LTX-2.3", + checkpoint_filename="ltx-2.3-22b-dev.safetensors", + checkpoint_revision="4229404625088d21c4f112eb640fb04a0900ee25", + checkpoint_env="VLLM_TEST_LTX23_OFFICIAL_CHECKPOINT", + stg_block=28, + image_repo="huggingface/documentation-images", + image_filename="diffusers/svd/rocket.png", + image_revision="645d8364f0c7a101180b364811b5a11a362e4010", + ), +) + + +def _run(command: list[str], *, env: dict[str, str], timeout: int = 1800) -> None: + start = time.perf_counter() + subprocess.run(command, env=env, timeout=timeout, check=True) + print(f"{' '.join(command[:3])} finished in {time.perf_counter() - start:.1f}s") + + +def _clone_official_source(root: Path, revision: str) -> None: + root.mkdir(parents=True) + repository = os.environ.get("VLLM_TEST_LTX_OFFICIAL_REPOSITORY", OFFICIAL_REPOSITORY) + subprocess.run(["git", "init", "-q", str(root)], check=True) + subprocess.run(["git", "-C", str(root), "remote", "add", "origin", repository], check=True) + last_error: subprocess.CalledProcessError | None = None + for attempt in range(3): + try: + subprocess.run( + ["git", "-C", str(root), "fetch", "--depth", "1", "origin", revision], + check=True, + ) + last_error = None + break + except subprocess.CalledProcessError as error: + last_error = error + if attempt < 2: + time.sleep(5 * (attempt + 1)) + if last_error is not None: + raise last_error + subprocess.run(["git", "-C", str(root), "checkout", "-q", "--detach", "FETCH_HEAD"], check=True) + + +def _git_revision(root: Path) -> str | None: + try: + return subprocess.check_output( + ["git", "-C", str(root), "rev-parse", "HEAD"], + stderr=subprocess.DEVNULL, + text=True, + ).strip() + except subprocess.CalledProcessError: + return None + + +def _official_source(artifact_root: Path) -> tuple[Path, str]: + revision = os.environ.get("VLLM_TEST_LTX_OFFICIAL_REVISION", OFFICIAL_REVISION) + configured_root = os.environ.get("VLLM_TEST_LTX_OFFICIAL_ROOT") + root = Path(configured_root) if configured_root else artifact_root / f"official-source-{revision[:12]}" + actual_revision = _git_revision(root) if root.exists() else None + if actual_revision != revision and configured_root: + raise AssertionError(f"Official source revision mismatch: {actual_revision} != {revision}") + if actual_revision != revision: + if root.exists(): + shutil.rmtree(root) + _clone_official_source(root, revision) + actual_revision = _git_revision(root) + assert actual_revision == revision, f"Official source revision mismatch: {actual_revision} != {revision}" + return root, revision + + +def _official_runner_prefix() -> list[str]: + """Run the reference with its missing binary dependency isolated from CI.""" + uv = shutil.which("uv") + assert uv is not None, "uv is required to run the pinned official LTX reference" + return [ + uv, + "run", + "--no-project", + "--with", + f"openimageio=={OFFICIAL_OPENIMAGEIO_VERSION}", + "--python", + sys.executable, + "python", + ] + + +def _resolve_model(case: LTXAccuracyCase) -> Path: + configured_model = os.environ.get(case.model_env) + if configured_model and Path(configured_model).exists(): + return Path(configured_model) + model_id = configured_model or case.model_id + revision = os.environ.get(f"{case.model_env}_REVISION") + if revision is None and model_id == case.model_id: + revision = case.model_revision + return Path( + snapshot_download( + repo_id=model_id, + revision=revision, + allow_patterns=[ + "model_index.json", + "audio_vae/*", + "connectors/*", + "processor/*", + "scheduler/*", + "text_encoder/config.json", + "text_encoder/generation_config.json", + "text_encoder/model*", + "tokenizer/*", + "transformer/*", + "vae/*", + "vocoder/*", + ], + ) + ) + + +def _resolve_gemma_root(model: Path) -> Path: + configured_root = os.environ.get("VLLM_TEST_LTX_GEMMA_ROOT") + if configured_root: + root = Path(configured_root) + assert root.is_dir(), f"Gemma root not found: {root}" + return root + return model + + +def _resolve_checkpoint(case: LTXAccuracyCase, model: Path) -> Path: + configured_checkpoint = os.environ.get(case.checkpoint_env) + if configured_checkpoint: + checkpoint = Path(configured_checkpoint) + assert checkpoint.is_file(), f"Official checkpoint not found: {checkpoint}" + return checkpoint + model_checkpoint = model / case.checkpoint_filename + if model_checkpoint.is_file(): + return model_checkpoint + return Path( + hf_hub_download( + repo_id=case.checkpoint_repo, + filename=case.checkpoint_filename, + revision=case.checkpoint_revision, + ) + ) + + +def _resolve_image(case: LTXAccuracyCase) -> Path | None: + if case.image_filename is None: + return None + if case.image_repo is None or case.image_revision is None: + raise ValueError(f"Incomplete image source for LTX accuracy case {case.name!r}.") + configured_image = os.environ.get("VLLM_TEST_LTX_I2V_IMAGE") + if configured_image: + image = Path(configured_image) + assert image.is_file(), f"LTX I2V conditioning image not found: {image}" + return image + return Path( + hf_hub_download( + repo_id=case.image_repo, + repo_type="dataset", + filename=case.image_filename, + revision=case.image_revision, + ) + ) + + +def _request(case: LTXAccuracyCase, image: Path | None) -> dict[str, object]: + request: dict[str, object] = { + "prompt": case.prompt, + "negative_prompt": NEGATIVE_PROMPT, + "width": 512, + "height": 384, + "num_frames": 25, + "fps": 24, + "num_inference_steps": 20, + "seed": 42, + "video_cfg_scale": 3.0, + "audio_cfg_scale": 7.0, + "video_stg_scale": 1.0, + "audio_stg_scale": 1.0, + "video_modality_scale": 3.0, + "audio_modality_scale": 3.0, + "video_rescale_scale": 0.7, + "audio_rescale_scale": 0.7, + "video_stg_blocks": [case.stg_block], + "audio_stg_blocks": [case.stg_block], + } + if image is not None: + request["image"] = str(image.resolve()) + return request + + +def _video_metrics(reference: np.ndarray, prediction: np.ndarray) -> dict[str, float]: + assert reference.shape == prediction.shape + assert reference.ndim == 4 and reference.shape[-1] == 3 + ssim_scores: list[float] = [] + psnr_scores: list[float] = [] + for reference_frame, prediction_frame in zip(reference, prediction, strict=True): + reference_tensor = torch.from_numpy(reference_frame).permute(2, 0, 1).unsqueeze(0) + prediction_tensor = torch.from_numpy(prediction_frame).permute(2, 0, 1).unsqueeze(0) + ssim_scores.append(float(StructuralSimilarityIndexMeasure(data_range=1.0)(prediction_tensor, reference_tensor))) + psnr_scores.append(float(PeakSignalNoiseRatio(data_range=1.0)(prediction_tensor, reference_tensor))) + difference = np.abs(reference.astype(np.float64) - prediction.astype(np.float64)) + return { + "ssim_mean": float(np.mean(ssim_scores)), + "ssim_min": float(np.min(ssim_scores)), + "psnr_mean_db": float(np.mean(psnr_scores)), + "max_abs": float(difference.max()), + "mean_abs": float(difference.mean()), + } + + +def _canonical_audio(audio: np.ndarray) -> np.ndarray: + while audio.ndim > 2 and audio.shape[0] == 1: + audio = audio[0] + return audio.astype(np.float64) + + +def _audio_metrics(reference: np.ndarray, prediction: np.ndarray) -> dict[str, float | bool]: + reference = _canonical_audio(reference) + prediction = _canonical_audio(prediction) + assert reference.shape == prediction.shape + difference = reference - prediction + reference_norm = max(float(np.linalg.norm(reference)), 1e-12) + prediction_norm = max(float(np.linalg.norm(prediction)), 1e-12) + return { + "bitwise_equal": bool(np.array_equal(reference, prediction)), + "max_abs": float(np.abs(difference).max()), + "mean_abs": float(np.abs(difference).mean()), + "relative_l2": float(np.linalg.norm(difference) / reference_norm), + "cosine_similarity": float(np.vdot(reference.ravel(), prediction.ravel()) / (reference_norm * prediction_norm)), + } + + +@pytest.mark.parametrize("case", CASES, ids=lambda case: case.name) +@pytest.mark.full_model +@pytest.mark.benchmark +@pytest.mark.diffusion +@hardware_test(res={"cuda": "H100"}, num_cards=1) +def test_ltx_one_stage_matches_official(case: LTXAccuracyCase, accuracy_artifact_root: Path) -> None: + """Compare official and Omni raw AV outputs from the same E2E request.""" + output_root = reset_artifact_dir(accuracy_artifact_root / "ltx_official" / case.name) + official_root, official_revision = _official_source(accuracy_artifact_root / "ltx_official") + model = _resolve_model(case) + gemma_root = _resolve_gemma_root(model) + checkpoint = _resolve_checkpoint(case, model) + image = _resolve_image(case) + request_path = output_root / "request.json" + request_path.write_text(json.dumps(_request(case, image), indent=2) + "\n") + + runner = Path(__file__).with_name("run_ltx_reference.py") + runner_args = [ + str(runner), + "--request", + str(request_path), + ] + if os.environ.get("VLLM_TEST_LTX_ENABLE_LAYERWISE_OFFLOAD", "").lower() in {"1", "true", "yes", "on"}: + runner_args.append("--enable-layerwise-offload") + env = os.environ.copy() + env["VLLM_TEST_LTX_OFFICIAL_REVISION"] = official_revision + env["PYTHONUNBUFFERED"] = "1" + repository_root = Path(__file__).resolve().parents[4] + existing_pythonpath = env.get("PYTHONPATH") + env["PYTHONPATH"] = ( + str(repository_root) if not existing_pythonpath else f"{repository_root}{os.pathsep}{existing_pythonpath}" + ) + + official_output = output_root / "official" + _run( + _official_runner_prefix() + + runner_args + + [ + "--backend", + "official", + "--output-dir", + str(official_output), + "--official-root", + str(official_root), + "--checkpoint", + str(checkpoint), + "--gemma-root", + str(gemma_root), + ], + env=env, + ) + + omni_output = output_root / "omni" + _run( + [sys.executable] + + runner_args + + [ + "--backend", + "omni", + "--output-dir", + str(omni_output), + "--model", + str(model), + "--model-class-name", + case.model_class_name, + ], + env=env, + ) + + official_metadata = json.loads((official_output / "metadata.json").read_text()) + omni_metadata = json.loads((omni_output / "metadata.json").read_text()) + assert official_metadata["attention_backend"] == ATTENTION_BACKEND + assert omni_metadata["attention_backend"] == ATTENTION_BACKEND + assert official_metadata["audio_sample_rate"] == omni_metadata["audio_sample_rate"] + video_metrics = _video_metrics( + np.load(official_output / "video.npy"), + np.load(omni_output / "video.npy"), + ) + audio_metrics = _audio_metrics( + np.load(official_output / "audio.npy"), + np.load(omni_output / "audio.npy"), + ) + result = { + "case": case.name, + "task": "i2v" if image is not None else "t2v", + "attention_backend": ATTENTION_BACKEND, + "official_revision": official_revision, + "model_revision": case.model_revision, + "checkpoint_revision": case.checkpoint_revision, + "video": video_metrics, + "audio": audio_metrics, + } + (output_root / "metrics.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result, indent=2)) + + assert video_metrics["ssim_mean"] >= VIDEO_SSIM_MEAN_THRESHOLD + assert video_metrics["ssim_min"] >= VIDEO_SSIM_MIN_THRESHOLD + assert video_metrics["psnr_mean_db"] >= VIDEO_PSNR_MEAN_THRESHOLD + assert audio_metrics["relative_l2"] <= AUDIO_RELATIVE_L2_THRESHOLD + assert audio_metrics["cosine_similarity"] >= AUDIO_COSINE_THRESHOLD diff --git a/tests/e2e/accuracy/test_ltx2_3_video_similarity.py b/tests/e2e/accuracy/test_ltx2_3_video_similarity.py deleted file mode 100644 index a169817a2..000000000 --- a/tests/e2e/accuracy/test_ltx2_3_video_similarity.py +++ /dev/null @@ -1,413 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -""" -SSIM/PSNR accuracy tests for LTX-2.3. - -1. **Transformer parity** (``test_ltx2_3_transformer_matches_diffusers``): - Swaps our custom transformer into diffusers' ``LTX2Pipeline`` to measure - numerical parity in isolation. Thresholds: SSIM >= 0.95, PSNR >= 28 dB. - Result: SSIM 0.999987 (bit-identical). - -2. **Full pipeline** (``test_ltx2_3_pipeline_matches_diffusers``): - Runs the full vLLM-Omni serving stack (``OmniServer`` -> HTTP API) and - compares per-frame against stock diffusers. Currently skipped because - the OmniServer subprocess creates a different RNG state than in-process - diffusers, producing different initial latents from the same seed. - This is a test infrastructure limitation, not a model accuracy issue. -""" - -from __future__ import annotations - -import gc -import os -import tempfile -from pathlib import Path - -import diffusers -import numpy as np -import pytest -import requests -import torch -from PIL import Image - -from tests.e2e.accuracy.helpers import compute_image_ssim_psnr, model_output_dir -from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup -from tests.helpers.mark import hardware_test -from tests.helpers.runtime import OmniServer - -# Parse diffusers version for compatibility check -_DIFFUSERS_VERSION = tuple(int(x) for x in diffusers.__version__.split(".")[:2] if x.isdigit()) -_DIFFUSERS_038 = _DIFFUSERS_VERSION >= (0, 38) - -MODEL_ID = "diffusers/LTX-2.3-Diffusers" -MODEL_ENV_VAR = "VLLM_TEST_LTX23_MODEL" -PROMPT = "A lighthouse on a rocky cliff at sunset, waves crashing below, golden hour lighting" -NEGATIVE_PROMPT = "blurry, low quality, distorted, watermark" -WIDTH = 512 -HEIGHT = 384 -NUM_FRAMES = 25 # ~1 second at 24fps -NUM_INFERENCE_STEPS = 20 -GUIDANCE_SCALE = 4.0 -SEED = 42 - -# Transformer-swap test: near-identical output expected -TRANSFORMER_SSIM_THRESHOLD = 0.95 -TRANSFORMER_PSNR_THRESHOLD = 28.0 - -# Full-pipeline test: allows minor divergence from RNG / pipeline differences -PIPELINE_SSIM_THRESHOLD = 0.94 -PIPELINE_PSNR_THRESHOLD = 28.0 - - -def _model_name() -> str: - return os.environ.get(MODEL_ENV_VAR, MODEL_ID) - - -def _local_files_only(model: str) -> bool: - return Path(model).exists() - - -# --------------------------------------------------------------------------- -# Frame extraction helpers -# --------------------------------------------------------------------------- - - -def _video_to_frames(video_np: np.ndarray) -> list[Image.Image]: - """Convert numpy video to list of PIL Images.""" - while video_np.ndim > 4: - video_np = video_np[0] - if video_np.dtype in (np.float32, np.float64, np.float16): - video_np = np.clip(video_np * 255, 0, 255).astype(np.uint8) - return [Image.fromarray(video_np[t]) for t in range(video_np.shape[0])] - - -def _extract_diffusers_frames(result) -> list[Image.Image]: - """Extract frames from diffusers pipeline output.""" - video = result.frames - if isinstance(video, np.ndarray): - return _video_to_frames(video) - if isinstance(video, list): - if isinstance(video[0], list): - return [img.convert("RGB") for img in video[0]] - if isinstance(video[0], Image.Image): - return [img.convert("RGB") for img in video] - raise ValueError(f"Unexpected output type: {type(video)}") - - -def _extract_mp4_frames(mp4_bytes: bytes) -> list[Image.Image]: - """Extract frames from an MP4 video using ffmpeg.""" - import subprocess - - with tempfile.TemporaryDirectory() as tmpdir: - mp4_path = os.path.join(tmpdir, "video.mp4") - with open(mp4_path, "wb") as f: - f.write(mp4_bytes) - - # Extract video frames as PNG files using ffmpeg - frame_pattern = os.path.join(tmpdir, "frame_%04d.png") - subprocess.run( - ["ffmpeg", "-i", mp4_path, "-vsync", "0", frame_pattern], - capture_output=True, - check=True, - ) - - # Load frames in order - frames = [] - i = 1 - while True: - fpath = os.path.join(tmpdir, f"frame_{i:04d}.png") - if not os.path.exists(fpath): - break - frames.append(Image.open(fpath).convert("RGB").copy()) - i += 1 - return frames - - -# --------------------------------------------------------------------------- -# Comparison helper -# --------------------------------------------------------------------------- - - -def _assert_video_similarity( - *, - model_name: str, - vllm_frames: list[Image.Image], - diffusers_frames: list[Image.Image], - ssim_threshold: float, - psnr_threshold: float, -) -> tuple[float, float]: - """Compare video frames and assert SSIM/PSNR meet thresholds.""" - min_frames = min(len(vllm_frames), len(diffusers_frames)) - assert min_frames > 0, "No frames to compare" - - ssim_scores = [] - psnr_scores = [] - for i in range(min_frames): - ssim_val, psnr_val = compute_image_ssim_psnr( - prediction=vllm_frames[i], - reference=diffusers_frames[i], - ) - ssim_scores.append(ssim_val) - psnr_scores.append(psnr_val) - - avg_ssim = sum(ssim_scores) / len(ssim_scores) - avg_psnr = sum(psnr_scores) / len(psnr_scores) - - print(f"\n{model_name} video similarity ({min_frames} frames):") - print(f" SSIM: avg={avg_ssim:.6f}, min={min(ssim_scores):.6f}, threshold>={ssim_threshold:.6f}") - print(f" PSNR: avg={avg_psnr:.6f} dB, min={min(psnr_scores):.6f} dB, threshold>={psnr_threshold:.6f} dB") - - assert avg_ssim >= ssim_threshold, f"SSIM below threshold: got {avg_ssim:.6f}, expected >= {ssim_threshold:.6f}." - assert avg_psnr >= psnr_threshold, f"PSNR below threshold: got {avg_psnr:.6f}, expected >= {psnr_threshold:.6f}." - return avg_ssim, avg_psnr - - -# --------------------------------------------------------------------------- -# Diffusers baseline (shared by both tests) -# --------------------------------------------------------------------------- - - -def _run_diffusers_baseline(model: str, output_dir: Path) -> list[Image.Image]: - """Generate video using stock diffusers LTX2Pipeline.""" - from diffusers import LTX2Pipeline - - run_pre_test_cleanup() - pipe = None - try: - pipe = LTX2Pipeline.from_pretrained( - model, torch_dtype=torch.bfloat16, local_files_only=_local_files_only(model) - ).to("cuda") - - generator = torch.Generator(device="cuda").manual_seed(SEED) - result = pipe( - prompt=PROMPT, - negative_prompt=NEGATIVE_PROMPT, - width=WIDTH, - height=HEIGHT, - num_frames=NUM_FRAMES, - num_inference_steps=NUM_INFERENCE_STEPS, - guidance_scale=GUIDANCE_SCALE, - generator=generator, - output_type="np", - ) - frames = _extract_diffusers_frames(result) - for i, f in enumerate(frames): - f.save(output_dir / f"diffusers_frame_{i:04d}.png") - return frames - finally: - del pipe - gc.collect() - if torch.cuda.is_available(): - torch.accelerator.empty_cache() - run_post_test_cleanup() - - -# --------------------------------------------------------------------------- -# Test 1: Transformer-swap parity -# --------------------------------------------------------------------------- - - -def _run_with_custom_transformer(model: str, output_dir: Path) -> list[Image.Image]: - """Run diffusers pipeline with our custom transformer swapped in.""" - from contextlib import nullcontext - - from diffusers import LTX2Pipeline - from vllm.config import VllmConfig, set_current_vllm_config - from vllm.distributed.parallel_state import init_distributed_environment, initialize_model_parallel - - from vllm_omni.diffusion.models.ltx2 import ( - create_transformer_from_config, - load_transformer_config, - ) - - vllm_config = VllmConfig() - ctx = set_current_vllm_config(vllm_config) - ctx.__enter__() - - if not torch.distributed.is_initialized(): - os.environ.setdefault("MASTER_ADDR", "localhost") - os.environ.setdefault("MASTER_PORT", "29503") - os.environ.setdefault("RANK", "0") - os.environ.setdefault("WORLD_SIZE", "1") - init_distributed_environment(world_size=1, rank=0, local_rank=0) - initialize_model_parallel(tensor_model_parallel_size=1) - - local = _local_files_only(model) - pipe = LTX2Pipeline.from_pretrained(model, torch_dtype=torch.bfloat16, local_files_only=local) - - transformer_config = load_transformer_config(model, "transformer", local) - our_transformer = create_transformer_from_config(transformer_config) - - diffusers_state = dict(pipe.transformer.named_parameters()) - - def _weight_iter(): - for name, param in diffusers_state.items(): - yield name, param.data - - our_transformer.load_weights(_weight_iter()) - our_transformer = our_transformer.to(dtype=torch.bfloat16, device="cuda").eval() - - # Compatibility shims for diffusers pipeline - our_transformer.dtype = torch.bfloat16 - if not hasattr(our_transformer, "cache_context"): - our_transformer.cache_context = lambda name: nullcontext() - - del pipe.transformer - pipe.transformer = our_transformer - for name, component in pipe.components.items(): - if name != "transformer" and hasattr(component, "to"): - try: - component.to("cuda") - except Exception: - pass - - generator = torch.Generator(device="cuda").manual_seed(SEED) - result = pipe( - prompt=PROMPT, - negative_prompt=NEGATIVE_PROMPT, - width=WIDTH, - height=HEIGHT, - num_frames=NUM_FRAMES, - num_inference_steps=NUM_INFERENCE_STEPS, - guidance_scale=GUIDANCE_SCALE, - generator=generator, - output_type="np", - ) - frames = _extract_diffusers_frames(result) - for i, f in enumerate(frames): - f.save(output_dir / f"vllm_transformer_frame_{i:04d}.png") - - del pipe, result, our_transformer - gc.collect() - if torch.cuda.is_available(): - torch.accelerator.empty_cache() - return frames - - -@pytest.mark.full_model -@pytest.mark.benchmark -@pytest.mark.diffusion -@pytest.mark.skipif( - not _DIFFUSERS_038, reason="LTX-2.3 requires diffusers >= 0.38.0 for cross_attn_mod and BWE vocoder" -) -@hardware_test(res={"cuda": "H100"}, num_cards=1) -def test_ltx2_3_transformer_matches_diffusers(accuracy_artifact_root: Path) -> None: - """Transformer-level parity: swap our transformer into diffusers pipeline. - - Isolates transformer numerical accuracy from pipeline-level differences. - Both runs use diffusers' denoising loop, CFG, scheduler, and RNG. - """ - model = _model_name() - output_dir = model_output_dir(accuracy_artifact_root, MODEL_ID) - - diffusers_frames = _run_diffusers_baseline(model=model, output_dir=output_dir) - vllm_frames = _run_with_custom_transformer(model=model, output_dir=output_dir) - - _assert_video_similarity( - model_name=f"{MODEL_ID} (transformer-swap)", - vllm_frames=vllm_frames, - diffusers_frames=diffusers_frames, - ssim_threshold=TRANSFORMER_SSIM_THRESHOLD, - psnr_threshold=TRANSFORMER_PSNR_THRESHOLD, - ) - - -# --------------------------------------------------------------------------- -# Test 2: Full pipeline (OmniServer → HTTP API vs diffusers) -# --------------------------------------------------------------------------- - - -def _run_vllm_omni_serving(model: str, output_dir: Path) -> list[Image.Image]: - """Generate video via the full vLLM-Omni serving stack.""" - server_args = [ - "--model-class-name", - "LTX23Pipeline", - "--stage-init-timeout", - "600", - ] - with OmniServer(model, server_args, use_omni=True) as server: - # Submit generation request - response = requests.post( - f"http://{server.host}:{server.port}/v1/videos", - files={ - "prompt": (None, PROMPT), - "negative_prompt": (None, NEGATIVE_PROMPT), - "model": (None, server.model), - "num_frames": (None, str(NUM_FRAMES)), - "fps": (None, "24"), - "size": (None, f"{WIDTH}x{HEIGHT}"), - "num_inference_steps": (None, str(NUM_INFERENCE_STEPS)), - "guidance_scale": (None, str(GUIDANCE_SCALE)), - "seed": (None, str(SEED)), - }, - timeout=120, - ) - response.raise_for_status() - video_id = response.json()["id"] - - # Poll for completion - import time - - for _ in range(120): - status_resp = requests.get( - f"http://{server.host}:{server.port}/v1/videos/{video_id}", - timeout=30, - ) - status_resp.raise_for_status() - status = status_resp.json()["status"] - if status == "completed": - break - if status in ("error", "failed"): - raise RuntimeError(f"Video generation failed: {status_resp.json()}") - time.sleep(5) - else: - raise TimeoutError(f"Video generation timed out after 600s (id={video_id})") - - # Download video content - content_resp = requests.get( - f"http://{server.host}:{server.port}/v1/videos/{video_id}/content", - timeout=120, - ) - content_resp.raise_for_status() - mp4_bytes = content_resp.content - - # Save MP4 - mp4_path = output_dir / "vllm_omni_pipeline.mp4" - with open(mp4_path, "wb") as f: - f.write(mp4_bytes) - - # Extract frames - frames = _extract_mp4_frames(mp4_bytes) - for i, frame in enumerate(frames): - frame.save(output_dir / f"vllm_pipeline_frame_{i:04d}.png") - return frames - - -@pytest.mark.full_model -@pytest.mark.benchmark -@pytest.mark.diffusion -@pytest.mark.skipif( - not _DIFFUSERS_038, reason="LTX-2.3 requires diffusers >= 0.38.0 for cross_attn_mod and BWE vocoder" -) -@hardware_test(res={"cuda": "H100"}, num_cards=1) -def test_ltx2_3_pipeline_matches_diffusers(accuracy_artifact_root: Path) -> None: - """Full-pipeline parity: vLLM-Omni serving stack vs diffusers. - - Runs the complete vLLM-Omni OmniServer (subprocess, HTTP API, video - encoding) and compares per-frame against stock diffusers output. - Follows the Wan2.2 / Qwen Image pattern with seed-based determinism. - """ - model = _model_name() - output_dir = model_output_dir(accuracy_artifact_root, MODEL_ID) - - diffusers_frames = _run_diffusers_baseline(model=model, output_dir=output_dir) - vllm_frames = _run_vllm_omni_serving(model=model, output_dir=output_dir) - - _assert_video_similarity( - model_name=f"{MODEL_ID} (full-pipeline)", - vllm_frames=vllm_frames, - diffusers_frames=diffusers_frames, - ssim_threshold=PIPELINE_SSIM_THRESHOLD, - psnr_threshold=PIPELINE_PSNR_THRESHOLD, - ) diff --git a/tests/e2e/online_serving/test_ltx2_expansion.py b/tests/e2e/online_serving/test_ltx2_expansion.py index 82855f62f..4d0eb6c90 100644 --- a/tests/e2e/online_serving/test_ltx2_expansion.py +++ b/tests/e2e/online_serving/test_ltx2_expansion.py @@ -4,8 +4,7 @@ Expansion tests for LTX-2 two-stage pipelines. Coverage: -- HSDP with LTX2TwoStagesPipeline (text-to-video) -- HSDP with LTX2ImageToVideoTwoStagesPipeline (image-to-video) +- HSDP with LTX2DistilledPipeline (text-to-video and image-to-video) """ import os @@ -27,7 +26,6 @@ HSDP_ARGS = ["--use-hsdp", "--hsdp-shard-size", "2"] def _cases(): cases = [] - # T2V: LTX2TwoStagesPipeline cases.append( pytest.param( OmniServerParams( @@ -35,15 +33,15 @@ def _cases(): server_args=[ *HSDP_ARGS, "--model-class-name", - "LTX2TwoStagesPipeline", + "LTX2DistilledPipeline", ], ), + False, id="t2v_hsdp", marks=PARALLEL_MARKS, ) ) - # I2V: LTX2ImageToVideoTwoStagesPipeline cases.append( pytest.param( OmniServerParams( @@ -51,9 +49,10 @@ def _cases(): server_args=[ *HSDP_ARGS, "--model-class-name", - "LTX2ImageToVideoTwoStagesPipeline", + "LTX2DistilledPipeline", ], ), + True, id="i2v_hsdp", marks=PARALLEL_MARKS, ) @@ -62,25 +61,22 @@ def _cases(): return cases -@pytest.mark.parametrize("omni_server", _cases(), indirect=True) +@pytest.mark.parametrize(("omni_server", "is_i2v"), _cases(), indirect=["omni_server"]) def test_ltx2_two_stage_hsdp( omni_server: OmniServer, + is_i2v: bool, openai_client: OpenAIClientHandler, ): - is_i2v = any("ImageToVideo" in arg for arg in omni_server.serve_args) - - # The two-stage pipeline generates at the requested resolution then 2x - # upsamples via LTX2LatentUpsamplerModel, so output dimensions differ - # from the request. Omit height/width from form_data so the assertion - # helper skips the dimension check (it only asserts when the key is - # present). The pipeline falls back to its own defaults. + # Keep CI small while exercising the fixed 8+3-step distilled recipe. + # Height and width describe the final output; Stage 1 runs at half size. form_data = { "prompt": PROMPT, "model": omni_server.model, + "height": 128, + "width": 128, "num_frames": 9, "fps": 8, - "num_inference_steps": 2, - "guidance_scale": 1.0, + "num_inference_steps": 8, "seed": 42, } diff --git a/tests/examples/offline_inference/test_ltx2_cfg_parallel_parity.py b/tests/examples/offline_inference/test_ltx2_cfg_parallel_parity.py deleted file mode 100644 index 757d906ec..000000000 --- a/tests/examples/offline_inference/test_ltx2_cfg_parallel_parity.py +++ /dev/null @@ -1,243 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import hashlib -import os -import subprocess -import sys -from pathlib import Path - -import numpy as np -import pytest -from PIL import Image - -from tests.helpers.mark import hardware_test - -REPO_ROOT = Path(__file__).resolve().parents[3] -T2V_EXAMPLE = REPO_ROOT / "examples" / "offline_inference" / "text_to_video" / "text_to_video.py" -I2V_EXAMPLE = REPO_ROOT / "examples" / "offline_inference" / "image_to_video" / "image_to_video.py" - -T2V_PROMPT = ( - "At sunrise, a glowing paper lantern boat drifts through a narrow canal between mossy stone walls, " - "soft fog above the water, the camera slowly gliding forward as golden reflections shimmer across " - "the ripples, cinematic, realistic, highly detailed." -) -T2V_NEGATIVE_PROMPT = "worst quality, blurry, jittery motion, distorted, oversaturated, artifacts" -I2V_PROMPT = "A cinematic dolly shot of a boat drifting on calm water at sunset" -I2V_NEGATIVE_PROMPT = "worst quality, blurry, jittery motion" - -os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" - - -def _get_ltx2_model() -> str: - return "Lightricks/LTX-2" - - -def _md5(path: Path) -> str: - digest = hashlib.md5(usedforsecurity=False) - with path.open("rb") as f: - for chunk in iter(lambda: f.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def _make_deterministic_test_image(path: Path) -> None: - """Create a deterministic 256x256 test image for I2V tests.""" - rng = np.random.RandomState(42) - img = Image.fromarray(rng.randint(0, 255, (256, 256, 3), dtype=np.uint8)) - img.save(path) - - -def _run_and_check(cmd: list[str], env: dict, output_path: Path, expected_md5: str) -> None: - result = subprocess.run(cmd, cwd=REPO_ROOT, env=env, capture_output=True, text=True, check=False) - assert result.returncode == 0, ( - f"Command failed (exit {result.returncode}).\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}" - ) - generated_md5 = _md5(output_path) - assert generated_md5 == expected_md5, ( - f"Unexpected output md5: {generated_md5} != {expected_md5}.\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}" - ) - - -# ── T2V tests ── - - -@pytest.mark.full_model -@pytest.mark.diffusion -@pytest.mark.parallel -@pytest.mark.slow -@hardware_test(res={"cuda": "L4"}, num_cards=2) -def test_ltx2_t2v_cfg_parallel(tmp_path: Path): - """T2V with CFG=4.0, cfg-parallel-size=2.""" - output = tmp_path / "t2v_cfg4.mp4" - env = os.environ.copy() - env.setdefault("CUDA_VISIBLE_DEVICES", "0,1") - cmd = [ - sys.executable, - str(T2V_EXAMPLE), - "--model", - _get_ltx2_model(), - "--prompt", - T2V_PROMPT, - "--negative-prompt", - T2V_NEGATIVE_PROMPT, - "--height", - "256", - "--width", - "256", - "--num-frames", - "145", - "--num-inference-steps", - "6", - "--guidance-scale", - "4.0", - "--frame-rate", - "24", - "--fps", - "24", - "--seed", - "42", - "--cfg-parallel-size", - "2", - "--enforce-eager", - "--output", - str(output), - ] - _run_and_check(cmd, env, output, expected_md5="08e606b9c522fee4b6f30cee8b77db40") - - -@pytest.mark.full_model -@pytest.mark.diffusion -@pytest.mark.slow -@hardware_test(res={"cuda": "L4"}, num_cards=1) -def test_ltx2_t2v_no_cfg(tmp_path: Path): - """T2V with CFG=1.0 (no classifier-free guidance).""" - output = tmp_path / "t2v_nocfg.mp4" - env = os.environ.copy() - env.setdefault("CUDA_VISIBLE_DEVICES", "0") - cmd = [ - sys.executable, - str(T2V_EXAMPLE), - "--model", - _get_ltx2_model(), - "--prompt", - T2V_PROMPT, - "--height", - "256", - "--width", - "256", - "--num-frames", - "145", - "--num-inference-steps", - "6", - "--guidance-scale", - "1.0", - "--frame-rate", - "24", - "--fps", - "24", - "--seed", - "42", - "--enforce-eager", - "--output", - str(output), - ] - _run_and_check(cmd, env, output, expected_md5="a83994b94b6e67c54a524e0383c45ce8") - - -# ── I2V tests ── - - -@pytest.mark.full_model -@pytest.mark.diffusion -@pytest.mark.parallel -@pytest.mark.slow -@hardware_test(res={"cuda": "L4"}, num_cards=2) -def test_ltx2_i2v_cfg_parallel(tmp_path: Path): - """I2V with CFG=4.0, cfg-parallel-size=2.""" - test_image = tmp_path / "test_input.png" - _make_deterministic_test_image(test_image) - output = tmp_path / "i2v_cfg4.mp4" - env = os.environ.copy() - env.setdefault("CUDA_VISIBLE_DEVICES", "0,1") - cmd = [ - sys.executable, - str(I2V_EXAMPLE), - "--model", - _get_ltx2_model(), - "--model-class-name", - "LTX2ImageToVideoPipeline", - "--image", - str(test_image), - "--prompt", - I2V_PROMPT, - "--negative-prompt", - I2V_NEGATIVE_PROMPT, - "--height", - "256", - "--width", - "256", - "--num-frames", - "73", - "--num-inference-steps", - "6", - "--guidance-scale", - "4.0", - "--frame-rate", - "24", - "--fps", - "24", - "--seed", - "42", - "--cfg-parallel-size", - "2", - "--enforce-eager", - "--output", - str(output), - ] - _run_and_check(cmd, env, output, expected_md5="aed7e56084b36373244d8f839b16d115") - - -@pytest.mark.full_model -@pytest.mark.diffusion -@pytest.mark.slow -@hardware_test(res={"cuda": "L4"}, num_cards=1) -def test_ltx2_i2v_no_cfg(tmp_path: Path): - """I2V with CFG=1.0 (no classifier-free guidance).""" - test_image = tmp_path / "test_input.png" - _make_deterministic_test_image(test_image) - output = tmp_path / "i2v_nocfg.mp4" - env = os.environ.copy() - env.setdefault("CUDA_VISIBLE_DEVICES", "0") - cmd = [ - sys.executable, - str(I2V_EXAMPLE), - "--model", - _get_ltx2_model(), - "--model-class-name", - "LTX2ImageToVideoPipeline", - "--image", - str(test_image), - "--prompt", - I2V_PROMPT, - "--height", - "256", - "--width", - "256", - "--num-frames", - "73", - "--num-inference-steps", - "6", - "--guidance-scale", - "1.0", - "--frame-rate", - "24", - "--fps", - "24", - "--seed", - "42", - "--enforce-eager", - "--output", - str(output), - ] - _run_and_check(cmd, env, output, expected_md5="81b21ede12753e9e14a357a6c548b666") diff --git a/tests/model_extras/test_model_extras.py b/tests/model_extras/test_model_extras.py index 47c108235..c73dbdf1e 100644 --- a/tests/model_extras/test_model_extras.py +++ b/tests/model_extras/test_model_extras.py @@ -181,6 +181,36 @@ def test_magi_human_extra_registry_declares_request_and_response_params() -> Non assert should_init_extra_args_for_non_diffusion_stages("MagiHumanPipeline") is False +@pytest.mark.core_model +@pytest.mark.cpu +def test_ltx_extra_registry_declares_official_guidance_params() -> None: + expected = frozenset( + { + "video_cfg_scale", + "audio_cfg_scale", + "video_cfg_guidance_scale", + "audio_cfg_guidance_scale", + "video_stg_scale", + "audio_stg_scale", + "video_stg_guidance_scale", + "audio_stg_guidance_scale", + "video_modality_scale", + "audio_modality_scale", + "a2v_guidance_scale", + "v2a_guidance_scale", + "video_rescale_scale", + "audio_rescale_scale", + "video_stg_blocks", + "audio_stg_blocks", + } + ) + + assert get_extra_body_params("LTX2Pipeline") == expected + assert get_extra_output_params("LTX2Pipeline") == frozenset() + assert get_extra_body_params("LTX2DistilledPipeline") == expected + assert get_extra_output_params("LTX2DistilledPipeline") == frozenset() + + @pytest.mark.core_model @pytest.mark.cpu def test_ming_flash_omni_extra_registry_declares_request_and_response_params() -> None: diff --git a/tests/model_tests/diffusion/test_alignment.py b/tests/model_tests/diffusion/test_alignment.py index 9d0a4911a..29cfc6f21 100644 --- a/tests/model_tests/diffusion/test_alignment.py +++ b/tests/model_tests/diffusion/test_alignment.py @@ -30,13 +30,9 @@ EXCLUDED_MODELS = [ "OvisImagePipeline", "WanPipeline", "WanVACEPipeline", - "LTX2ImageToVideoPipeline", - "LTX2TwoStagesPipeline", - "LTX2ImageToVideoTwoStagesPipeline", + "LTX2DistilledPipeline", "LTX2T2VDMD2Pipeline", "LTX2I2VDMD2Pipeline", - "LTX23Pipeline", - "LTX23ImageToVideoPipeline", "StableAudioPipeline", "WanImageToVideoPipeline", "WanS2VPipeline", diff --git a/vllm_omni/diffusion/model_loader/diffusers_loader.py b/vllm_omni/diffusion/model_loader/diffusers_loader.py index 9864d8dbf..d7770c266 100644 --- a/vllm_omni/diffusion/model_loader/diffusers_loader.py +++ b/vllm_omni/diffusion/model_loader/diffusers_loader.py @@ -634,7 +634,7 @@ class DiffusersPipelineLoader: # Discover pipeline components (DiT, encoders, VAEs) via # ModuleDiscovery, which consults SupportsComponentDiscovery # when available and falls back to well-known attribute names. - # This supports nested pipelines (e.g. LTX2TwoStagesPipeline + # This supports nested pipelines (e.g. LTX2DistilledPipeline # where the transformer lives at "pipe.transformer"). discovered_modules = ModuleDiscovery.discover(model) diff --git a/vllm_omni/diffusion/models/ltx2/__init__.py b/vllm_omni/diffusion/models/ltx2/__init__.py index ca34e0390..300295bd8 100644 --- a/vllm_omni/diffusion/models/ltx2/__init__.py +++ b/vllm_omni/diffusion/models/ltx2/__init__.py @@ -9,28 +9,16 @@ from vllm_omni.diffusion.models.ltx2.ltx2_components import ( from vllm_omni.diffusion.models.ltx2.ltx2_transformer import LTX2VideoTransformer3DModel from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import ( LTX2I2VDMD2Pipeline, - LTX2ImageToVideoPipeline, LTX2Pipeline, LTX2T2VDMD2Pipeline, - LTX23ImageToVideoPipeline, - LTX23Pipeline, -) -from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_latent_upsample import LTX2LatentUpsamplePipeline -from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_two_stage import ( - LTX2ImageToVideoTwoStagesPipeline, - LTX2TwoStagesPipeline, ) +from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_two_stage import LTX2DistilledPipeline __all__ = [ "LTX2Pipeline", "LTX2T2VDMD2Pipeline", - "LTX2ImageToVideoPipeline", "LTX2I2VDMD2Pipeline", - "LTX2LatentUpsamplePipeline", - "LTX2TwoStagesPipeline", - "LTX2ImageToVideoTwoStagesPipeline", - "LTX23Pipeline", - "LTX23ImageToVideoPipeline", + "LTX2DistilledPipeline", "get_ltx2_post_process_func", "load_transformer_config", "create_transformer_from_config", diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_components.py b/vllm_omni/diffusion/models/ltx2/ltx2_components.py index cfe990a4e..54c866eb7 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_components.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_components.py @@ -7,6 +7,7 @@ from __future__ import annotations import inspect import json +import logging import os from dataclasses import dataclass from typing import TYPE_CHECKING, Any @@ -14,11 +15,14 @@ from typing import TYPE_CHECKING, Any import torch from diffusers import AutoencoderKLLTX2Audio, AutoencoderKLLTX2Video, FlowMatchEulerDiscreteScheduler from diffusers.pipelines.ltx2 import LTX2TextConnectors +from diffusers.pipelines.ltx2.latent_upsampler import LTX2LatentUpsamplerModel from diffusers.pipelines.ltx2.vocoder import LTX2Vocoder from diffusers.video_processor import VideoProcessor from huggingface_hub import hf_hub_download from transformers import AutoTokenizer, Gemma3ForConditionalGeneration +from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata +from vllm_omni.diffusion.attention.layer import Attention as OmniAttention from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_ltx2 import DistributedAutoencoderKLLTX2Video from vllm_omni.diffusion.distributed.utils import get_local_device from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader @@ -28,7 +32,12 @@ from vllm_omni.diffusion.offloader.module_collector import ModuleDiscovery if TYPE_CHECKING: from vllm.model_executor.layers.quantization.base_config import QuantizationConfig -from .ltx2_transformer import LTX2VideoTransformer3DModel +from .ltx2_transformer import ( + LTX2VideoTransformer3DModel, + apply_interleaved_rotary_emb, + apply_split_rotary_emb, + to_ltx_padding_mask, +) try: from diffusers.pipelines.ltx2.vocoder import LTX2VocoderWithBWE @@ -44,7 +53,9 @@ _LTX_COMPONENT_SUBFOLDERS = ( "audio_vae", "vocoder", "scheduler", + "latent_upsampler", ) +logger = logging.getLogger(__name__) @dataclass(frozen=True) @@ -81,6 +92,164 @@ LTX23_COMPONENT_PROFILE = LTXComponentProfile( vocoder_fallback_cls=LTX2Vocoder, ) +LTX2_DISTILLED_COMPONENT_PROFILE = LTXComponentProfile( + name="ltx2_distilled", + dit_modules=("transformer",), + encoder_modules=("text_encoder", "connectors"), + vae_modules=("vae", "audio_vae"), + resident_modules=("vocoder", "latent_upsampler"), + video_vae_cls=DistributedAutoencoderKLLTX2Video, +) + + +_COMPONENT_PROFILES: dict[tuple[str, str], LTXComponentProfile] = { + ("one_stage", "2"): LTX2_COMPONENT_PROFILE, + ("one_stage", "2.3"): LTX23_COMPONENT_PROFILE, + ("distilled_two_stage", "2"): LTX2_DISTILLED_COMPONENT_PROFILE, + ("dmd2", "2"): LTX2_COMPONENT_PROFILE, + ("dmd2", "2.3"): LTX23_COMPONENT_PROFILE, +} + + +def resolve_ltx_component_profile(pipeline_kind: str, model_version: str) -> LTXComponentProfile: + """Resolve component construction independently from execution recipes.""" + try: + return _COMPONENT_PROFILES[(pipeline_kind, model_version)] + except KeyError as exc: + raise ValueError(f"Unsupported LTX component kind/version: {pipeline_kind!r}/{model_version!r}.") from exc + + +def _load_ltx_metadata_json(model: str, filename: str) -> dict[str, Any]: + """Load small checkpoint metadata without relying on repository names.""" + if os.path.isdir(model): + path = os.path.join(model, filename) + if not os.path.isfile(path): + return {} + else: + try: + path = hf_hub_download(repo_id=model, filename=filename) + except Exception: + return {} + try: + with open(path) as config_file: + value = json.load(config_file) + except (OSError, TypeError, ValueError): + return {} + return value if isinstance(value, dict) else {} + + +def detect_ltx_model_version(model: str) -> str: + """Detect LTX-2 versus LTX-2.3 from checkpoint component metadata. + + Official checkpoints use ``model_version`` metadata. Diffusers repositories + expose the equivalent distinction through the BWE vocoder introduced by + LTX-2.3. Unknown conversions retain the official LTX-2 fallback. + """ + model_index = _load_ltx_metadata_json(model, "model_index.json") + if str(model_index.get("model_version", "")).startswith("2.3"): + return "2.3" + + vocoder_entry = model_index.get("vocoder") + if isinstance(vocoder_entry, (list, tuple)) and vocoder_entry: + vocoder_class = str(vocoder_entry[-1]) + elif isinstance(vocoder_entry, dict): + vocoder_class = str(vocoder_entry.get("_class_name", "")) + else: + vocoder_class = "" + if vocoder_class == "LTX2VocoderWithBWE": + return "2.3" + + vocoder_config = _load_ltx_metadata_json(model, "vocoder/config.json") + if str(vocoder_config.get("model_version", "")).startswith("2.3"): + return "2.3" + if vocoder_config.get("_class_name") == "LTX2VocoderWithBWE": + return "2.3" + + logger.info("Using LTX-2 defaults for checkpoint %s", model) + return "2" + + +class _LTXConnectorAttnProcessor: + """Preserve official connector math around Omni attention dispatch.""" + + def __init__(self, *, has_learned_registers: bool) -> None: + self.has_learned_registers = has_learned_registers + + def __call__( + self, + attn: Any, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor | None = None, + attention_mask: torch.Tensor | None = None, + query_rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None, + key_rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None, + ) -> torch.Tensor: + encoder_hidden_states = encoder_hidden_states if encoder_hidden_states is not None else hidden_states + gate_logits = attn.to_gate_logits(hidden_states) if attn.to_gate_logits is not None else None + + query = attn.norm_q(attn.to_q(hidden_states)) + key = attn.norm_k(attn.to_k(encoder_hidden_states)) + value = attn.to_v(encoder_hidden_states) + + if query_rotary_emb is not None: + key_rotary_emb = key_rotary_emb if key_rotary_emb is not None else query_rotary_emb + if attn.rope_type == "interleaved": + query = apply_interleaved_rotary_emb(query, query_rotary_emb) + key = apply_interleaved_rotary_emb(key, key_rotary_emb) + elif attn.rope_type == "split": + query = apply_split_rotary_emb(query, query_rotary_emb, head_dim=attn.head_dim) + key = apply_split_rotary_emb(key, key_rotary_emb, head_dim=attn.head_dim) + else: + raise ValueError(f"Unsupported LTX connector RoPE type: {attn.rope_type}") + + batch_size, _, inner_dim = query.shape + head_dim = inner_dim // attn.heads + kv_heads = attn.inner_kv_dim // attn.head_dim + query = query.view(batch_size, -1, attn.heads, head_dim) + key = key.view(batch_size, -1, kv_heads, head_dim) + value = value.view(batch_size, -1, kv_heads, head_dim) + + # The connector replaces padding tokens with learned registers before + # entering its blocks, so every key is valid and the old padding mask + # becomes an all-keep no-op. + if self.has_learned_registers: + attention_mask = None + elif attention_mask is not None and attn.omni_attention.attn_backend.get_name().upper() == "FLASH_ATTN": + attention_mask = to_ltx_padding_mask(attention_mask) + attn_metadata = AttentionMetadata(attn_mask=attention_mask) if attention_mask is not None else None + hidden_states = attn.omni_attention(query, key, value, attn_metadata) + hidden_states = hidden_states.reshape(batch_size, -1, inner_dim) + + if gate_logits is not None: + hidden_states = hidden_states.unflatten(2, (attn.heads, -1)) + hidden_states = hidden_states * (2.0 * torch.sigmoid(gate_logits)).unsqueeze(-1) + hidden_states = hidden_states.flatten(2, 3) + + hidden_states = attn.to_out[0](hidden_states) + return attn.to_out[1](hidden_states) + + +def _install_connector_attention(connectors: LTX2TextConnectors) -> None: + for connector_name in ("video_connector", "audio_connector"): + connector = getattr(connectors, connector_name, None) + has_learned_registers = getattr(connector, "learnable_registers", None) is not None + for block_index, block in enumerate(getattr(connector, "transformer_blocks", ())): + attention = getattr(block, "attn1", None) + if attention is not None: + attention.omni_attention = OmniAttention( + num_heads=attention.heads, + head_size=attention.head_dim, + num_kv_heads=attention.inner_kv_dim // attention.head_dim, + softmax_scale=1.0 / (attention.head_dim**0.5), + causal=False, + prefix=f"connectors.{connector_name}.transformer_blocks.{block_index}.attn1", + role="ltx2.connector", + role_category="self", + skip_sequence_parallel=True, + disable_kv_quant=True, + ) + attention.set_processor(_LTXConnectorAttnProcessor(has_learned_registers=has_learned_registers)) + def _detect_vocoder_output_sample_rate(model: str) -> int | None: """Read the generated waveform sample rate from the vocoder config.""" @@ -188,6 +357,7 @@ def initialize_pipeline_components(pipeline: Any, od_config: Any) -> None: local_files_only=local_files_only, dtype=dtype, ) + _install_connector_attention(pipeline.connectors) pipeline.vae = _load_component( profile.video_vae_cls, model, @@ -221,6 +391,18 @@ def initialize_pipeline_components(pipeline: Any, od_config: Any) -> None: dtype=dtype, ) + if "latent_upsampler" in profile.resident_modules: + # BlurDownsample constructs an integer kernel that must be initialized + # on CPU; component placement is handled uniformly after construction. + with torch.device("cpu"): + pipeline.latent_upsampler = _load_component( + LTX2LatentUpsamplerModel, + model, + "latent_upsampler", + local_files_only=local_files_only, + dtype=dtype, + ) + transformer_config = load_transformer_config(model, "transformer", local_files_only) quant_config = getattr(od_config, "quantization_config", None) pipeline.transformer = create_transformer_from_config(transformer_config, quant_config=quant_config) @@ -249,8 +431,6 @@ def initialize_pipeline_components(pipeline: Any, od_config: Any) -> None: tokenizer_max_length = getattr(encoder_config, "max_seq_len", None) pipeline.tokenizer_max_length = int(tokenizer_max_length or 1024) - pipeline._guidance_scale = None - pipeline._guidance_rescale = None pipeline._interrupt = False @@ -262,22 +442,15 @@ def load_transformer_config( """Load an LTX transformer config from a local model or the HF Hub.""" if local_files_only: config_path = os.path.join(model_path, subfolder, "config.json") - if os.path.exists(config_path): - with open(config_path) as config_file: - return json.load(config_file) + if not os.path.exists(config_path): + raise FileNotFoundError(f"LTX transformer config not found: {config_path}") else: - try: - from huggingface_hub import hf_hub_download - - config_path = hf_hub_download( - repo_id=model_path, - filename=f"{subfolder}/config.json", - ) - with open(config_path) as config_file: - return json.load(config_file) - except Exception: - pass - return {} + config_path = hf_hub_download( + repo_id=model_path, + filename=f"{subfolder}/config.json", + ) + with open(config_path) as config_file: + return json.load(config_file) def create_transformer_from_config( diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py b/vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py index 48d319f6d..6bd759057 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py @@ -5,17 +5,20 @@ from __future__ import annotations +import math from dataclasses import dataclass from typing import TYPE_CHECKING, Any +import numpy as np import PIL.Image import torch +import torch.nn.functional as F from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img import retrieve_latents from diffusers.utils.torch_utils import randn_tensor from diffusers.video_processor import VideoProcessor from . import ltx2_latents as latent_ops -from .ltx2_denoise import I2VVideoAudioScheduler +from .ltx2_guidance import euler_step_from_velocity if TYPE_CHECKING: from .ltx2_denoise import LTXDenoiseContext, LTXForwardContext @@ -44,6 +47,41 @@ def _repeat_prompt_tensor_for_outputs(tensor: torch.Tensor, num_outputs: int) -> return tensor.repeat_interleave(num_outputs, dim=0) +def _preprocess_i2v_pil_images( + images: PIL.Image.Image | list[PIL.Image.Image], + *, + height: int, + width: int, + device: torch.device | None = None, + dtype: torch.dtype | None = None, +) -> torch.Tensor: + """Match official LTX aspect-preserving resize and center crop.""" + image_list = images if isinstance(images, list) else [images] + processed = [] + for image in image_list: + pixels = ( + torch.from_numpy(np.array(image.convert("RGB"), copy=True)) + .permute(2, 0, 1) + .unsqueeze(0) + .to(device=device, dtype=torch.float32) + ) + source_height, source_width = pixels.shape[-2:] + scale = max(height / source_height, width / source_width) + resized_height = math.ceil(source_height * scale) + resized_width = math.ceil(source_width * scale) + pixels = F.interpolate( + pixels, + size=(resized_height, resized_width), + mode="bilinear", + align_corners=False, + ) + crop_top = (resized_height - height) // 2 + crop_left = (resized_width - width) // 2 + pixels = pixels[:, :, crop_top : crop_top + height, crop_left : crop_left + width] + processed.append((pixels / 127.5 - 1.0).to(dtype=dtype)) + return torch.cat(processed, dim=0) + + class LTXTextConditioningMixin: """Shared Gemma encoding and text connector orchestration.""" @@ -113,21 +151,64 @@ class LTXTextConditioningMixin: batch_size = len(prompt) if prompt is not None else prompt_embeds.shape[0] negative_prompt_embeds_provided = negative_prompt_embeds is not None - if prompt_embeds is None: - prompt_embeds, prompt_attention_mask = self._get_gemma_prompt_embeds( - prompt=prompt, - num_videos_per_prompt=num_videos_per_prompt, + if do_classifier_free_guidance and prompt_embeds is None and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type as `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + if isinstance(negative_prompt, list) and batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt` has batch size {len(negative_prompt)}, but `prompt` has batch size" + f" {batch_size}." + ) + + # Official LTX encodes positive and negative prompts in one Gemma batch; + # separate bf16 GEMMs produce a different conditioning trajectory. + combined_embeds, combined_mask = self._get_gemma_prompt_embeds( + prompt=prompt + negative_prompt, + num_videos_per_prompt=1, max_sequence_length=max_sequence_length, scale_factor=scale_factor, device=device, dtype=dtype, ) - elif num_videos_per_prompt > 1: - prompt_embeds = _repeat_prompt_tensor_for_outputs(prompt_embeds, num_videos_per_prompt) - prompt_attention_mask = _repeat_prompt_tensor_for_outputs( - prompt_attention_mask, - num_videos_per_prompt, - ) + prompt_embeds = combined_embeds[:batch_size] + negative_prompt_embeds = combined_embeds[batch_size:] + prompt_attention_mask = combined_mask[:batch_size] + negative_prompt_attention_mask = combined_mask[batch_size:] + if num_videos_per_prompt > 1: + prompt_embeds = _repeat_prompt_tensor_for_outputs(prompt_embeds, num_videos_per_prompt) + negative_prompt_embeds = _repeat_prompt_tensor_for_outputs( + negative_prompt_embeds, + num_videos_per_prompt, + ) + prompt_attention_mask = _repeat_prompt_tensor_for_outputs( + prompt_attention_mask, + num_videos_per_prompt, + ) + negative_prompt_attention_mask = _repeat_prompt_tensor_for_outputs( + negative_prompt_attention_mask, + num_videos_per_prompt, + ) + else: + if prompt_embeds is None: + prompt_embeds, prompt_attention_mask = self._get_gemma_prompt_embeds( + prompt=prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + scale_factor=scale_factor, + device=device, + dtype=dtype, + ) + elif num_videos_per_prompt > 1: + prompt_embeds = _repeat_prompt_tensor_for_outputs(prompt_embeds, num_videos_per_prompt) + prompt_attention_mask = _repeat_prompt_tensor_for_outputs( + prompt_attention_mask, + num_videos_per_prompt, + ) if do_classifier_free_guidance and negative_prompt_embeds is None: negative_prompt = negative_prompt or "" @@ -243,9 +324,10 @@ class LTXTextConditioningMixin: class LTXI2VConditioningMixin: - """First-frame conditioning behavior common to LTX2 and LTX2.3.""" + """First-frame conditioning with optional unified T2V/I2V dispatch.""" support_image_input = True + unified_text_image_entry = False def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -254,10 +336,6 @@ class LTXI2VConditioningMixin: resample="bilinear", ) - def forward(self, req: Any, image: Any | None = None, *args: Any, **kwargs: Any) -> Any: - """Preserve the public I2V positional API while sharing the T2V runner.""" - return super().forward(req, *args, image=image, **kwargs) - @staticmethod def _resolve_single_prompt_image(raw_image: Any) -> Any: if isinstance(raw_image, list): @@ -283,11 +361,12 @@ class LTXI2VConditioningMixin: image: Any | None, request_inputs: LTXRequestInputs, ) -> Any | None: - if image is not None or not req.prompts: + prompts = getattr(req, "prompts", None) + if image is not None or not prompts: return image raw_images = [] - for prompt_item in req.prompts: + for prompt_item in prompts: if isinstance(prompt_item, str): raw_image = None else: @@ -300,8 +379,12 @@ class LTXI2VConditioningMixin: raw_image = PIL.Image.open(raw_image).convert("RGB") raw_images.append(raw_image) + if all(raw_image is None for raw_image in raw_images) and self.unified_text_image_entry: + return None if any(raw_image is None for raw_image in raw_images) and request_inputs.latents is None: - raise ValueError("Image is required for LTX I2V generation.") + if not self.unified_text_image_entry: + raise ValueError("Image is required for LTX I2V generation.") + raise ValueError("LTX request batches cannot mix text-to-video and image-to-video prompts.") if len(raw_images) == 1: return raw_images[0] return raw_images or image @@ -311,75 +394,19 @@ class LTXI2VConditioningMixin: request_inputs: LTXRequestInputs, image: Any | None = None, ) -> None: - if image is None and request_inputs.latents is None: + if not self.unified_text_image_entry and image is None and request_inputs.latents is None: raise ValueError("Provide either `image` or `latents`. Cannot leave both undefined.") super()._check_forward_inputs(request_inputs, image=image) - def prepare_latents( + def _encode_i2v_image_latents( self, - image: torch.Tensor | None = None, - batch_size: int = 1, - num_channels_latents: int = 128, - height: int = 512, - width: int = 768, - num_frames: int = 121, - noise_scale: float = 0.0, - dtype: torch.dtype | None = None, - device: torch.device | None = None, - generator: torch.Generator | list[torch.Generator] | None = None, - latents: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, torch.Tensor]: - num_frames, height, width = latent_ops.resolve_video_latent_shape( - height, - width, - num_frames, - vae_spatial_compression_ratio=self.vae_spatial_compression_ratio, - vae_temporal_compression_ratio=self.vae_temporal_compression_ratio, - ) - shape = (batch_size, num_channels_latents, num_frames, height, width) - mask_shape = (batch_size, 1, num_frames, height, width) - - if latents is not None: - if latents.ndim == 5: - batch_size, _, num_frames, height, width = latents.shape - mask_shape = (batch_size, 1, num_frames, height, width) - conditioning_mask = latents.new_zeros(mask_shape) - conditioning_mask[:, :, 0] = 1.0 - latents = latent_ops.normalize_latents( - latents, - self.vae.latents_mean, - self.vae.latents_std, - self.vae.config.scaling_factor, - ) - latents = latent_ops.create_noised_state( - latents, - noise_scale * (1 - conditioning_mask), - generator, - ) - latents = latent_ops.pack_latents( - latents, - self.transformer_spatial_patch_size, - self.transformer_temporal_patch_size, - ) - else: - conditioning_mask = latents.new_zeros(mask_shape) - conditioning_mask[:, :, 0] = 1.0 - - conditioning_mask = latent_ops.pack_latents( - conditioning_mask, - self.transformer_spatial_patch_size, - self.transformer_temporal_patch_size, - ).squeeze(-1) - if latents.ndim != 3 or latents.shape[:2] != conditioning_mask.shape: - raise ValueError( - "Provided `latents` tensor has shape" - f" {latents.shape}, but the expected shape is {conditioning_mask.shape + (num_channels_latents,)}." - ) - return latents.to(device=device, dtype=dtype), conditioning_mask - - if image is None: - raise ValueError("`image` must be provided when `latents` is None.") - + image: torch.Tensor, + *, + batch_size: int, + generator: torch.Generator | list[torch.Generator] | None, + dtype: torch.dtype | None, + ) -> torch.Tensor: + """Encode one conditioning frame per prompt into normalized latents.""" image_batch_size = image.shape[0] if image_batch_size == 0: raise ValueError("`image` batch is empty.") @@ -415,38 +442,173 @@ class LTXI2VConditioningMixin: for img in image ] - init_latents = torch.cat(init_latents, dim=0).to(dtype) + init_latents = torch.cat(init_latents, dim=0).to(dtype=dtype) if num_videos_per_prompt > 1: init_latents = init_latents.repeat_interleave(num_videos_per_prompt, dim=0) - init_latents = latent_ops.normalize_latents( + return latent_ops.normalize_latents( init_latents, self.vae.latents_mean, self.vae.latents_std, ) + + @staticmethod + def _replace_first_latent_frame( + latents: torch.Tensor, + image_latents: torch.Tensor, + ) -> torch.Tensor: + expected_shape = (latents.shape[0], latents.shape[1], 1, latents.shape[3], latents.shape[4]) + if tuple(image_latents.shape) != expected_shape: + raise ValueError( + f"Encoded LTX conditioning image has shape {tuple(image_latents.shape)}, expected {expected_shape}." + ) + latents = latents.clone() + latents[:, :, :1] = image_latents + return latents + + def prepare_latents( + self, + image: torch.Tensor | None = None, + batch_size: int = 1, + num_channels_latents: int = 128, + height: int = 512, + width: int = 768, + num_frames: int = 121, + noise_scale: float = 0.0, + dtype: torch.dtype | None = None, + device: torch.device | None = None, + generator: torch.Generator | list[torch.Generator] | None = None, + latents: torch.Tensor | None = None, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + if image is None and self.unified_text_image_entry: + return super().prepare_latents( + batch_size=batch_size, + num_channels_latents=num_channels_latents, + height=height, + width=width, + num_frames=num_frames, + noise_scale=noise_scale, + dtype=dtype, + device=device, + generator=generator, + latents=latents, + ) + + num_frames, height, width = latent_ops.resolve_video_latent_shape( + height, + width, + num_frames, + vae_spatial_compression_ratio=self.vae_spatial_compression_ratio, + vae_temporal_compression_ratio=self.vae_temporal_compression_ratio, + ) + mask_shape = (batch_size, 1, num_frames, height, width) + + if latents is not None: + unpacked_latents = latents.ndim == 5 + if latents.ndim == 5: + expected_shape = (batch_size, num_channels_latents, num_frames, height, width) + if tuple(latents.shape) != expected_shape: + raise ValueError( + f"Provided unpacked `latents` have shape {tuple(latents.shape)}, expected {expected_shape}." + ) + conditioning_mask = latents.new_zeros(mask_shape) + conditioning_mask[:, :, 0] = 1.0 + latents = latent_ops.normalize_latents( + latents, + self.vae.latents_mean, + self.vae.latents_std, + self.vae.config.scaling_factor, + ) + if image is not None: + image_latents = self._encode_i2v_image_latents( + image, + batch_size=batch_size, + generator=generator, + dtype=dtype, + ) + # Official distilled Stage 2 starts from the upsampled Stage 1 + # state, then reapplies the source image at final resolution. + latents = self._replace_first_latent_frame(latents, image_latents) + latents = latent_ops.pack_latents( + latents, + self.transformer_spatial_patch_size, + self.transformer_temporal_patch_size, + ) + else: + conditioning_mask = latents.new_zeros(mask_shape) + conditioning_mask[:, :, 0] = 1.0 + + conditioning_mask = latent_ops.pack_latents( + conditioning_mask, + self.transformer_spatial_patch_size, + self.transformer_temporal_patch_size, + ).squeeze(-1) + if latents.ndim != 3 or latents.shape[:2] != conditioning_mask.shape: + raise ValueError( + "Provided `latents` tensor has shape" + f" {latents.shape}, but the expected shape is {conditioning_mask.shape + (num_channels_latents,)}." + ) + if unpacked_latents: + # Official LTX patchifies the latent state before drawing noise. + # Sampling the packed shape preserves its seed-to-output contract. + latents = latent_ops.create_noised_state( + latents, + noise_scale * (1 - conditioning_mask).unsqueeze(-1), + generator, + ) + elif image is not None: + image_latents = self._encode_i2v_image_latents( + image, + batch_size=batch_size, + generator=generator, + dtype=dtype, + ) + unpacked = latent_ops.unpack_latents( + latents, + num_frames, + height, + width, + self.transformer_spatial_patch_size, + self.transformer_temporal_patch_size, + ) + unpacked = self._replace_first_latent_frame(unpacked, image_latents) + latents = latent_ops.pack_latents( + unpacked, + self.transformer_spatial_patch_size, + self.transformer_temporal_patch_size, + ) + return latents.to(device=device, dtype=dtype), conditioning_mask + + if image is None: + raise ValueError("`image` must be provided when `latents` is None.") + init_latents = self._encode_i2v_image_latents( + image, + batch_size=batch_size, + generator=generator, + dtype=dtype, + ) init_latents = init_latents.repeat(1, 1, num_frames, 1, 1) conditioning_mask = torch.zeros(mask_shape, device=device, dtype=dtype) conditioning_mask[:, :, 0] = 1.0 - noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) - latents = init_latents * conditioning_mask + noise * (1 - conditioning_mask) - + init_latents = latent_ops.pack_latents( + init_latents, + self.transformer_spatial_patch_size, + self.transformer_temporal_patch_size, + ) conditioning_mask = latent_ops.pack_latents( conditioning_mask, self.transformer_spatial_patch_size, self.transformer_temporal_patch_size, ).squeeze(-1) - latents = latent_ops.pack_latents( - latents, - self.transformer_spatial_patch_size, - self.transformer_temporal_patch_size, - ) + noise = randn_tensor(init_latents.shape, generator=generator, device=device, dtype=dtype) + latents = init_latents * conditioning_mask.unsqueeze(-1) + noise * (1 - conditioning_mask).unsqueeze(-1) return latents, conditioning_mask def _step_video_latents_i2v( self, noise_pred_video: torch.Tensor, latents: torch.Tensor, - timestep: torch.Tensor, + step_index: int, latent_num_frames: int, latent_height: int, latent_width: int, @@ -469,12 +631,14 @@ class LTXI2VConditioningMixin: ) noise_pred_video = noise_pred_video[:, :, 1:] noise_latents = latents_unpacked[:, :, 1:] - pred_latents = self.scheduler.step( - noise_pred_video, - timestep, + # Official LTX advances the unconditioned frames directly on the + # sigma trajectory; the first frame remains fixed conditioning. + pred_latents = euler_step_from_velocity( noise_latents, - return_dict=False, - )[0] + noise_pred_video, + self.scheduler.sigmas, + step_index, + ) latents_unpacked = torch.cat([latents_unpacked[:, :, :1], pred_latents], dim=2) return latent_ops.pack_latents( latents_unpacked, @@ -491,12 +655,31 @@ class LTXI2VConditioningMixin: noise_scale: float, image: Any | None = None, ) -> tuple[torch.Tensor, torch.Tensor | None]: - if request_inputs.latents is None: + if image is None and self.unified_text_image_entry: + return super()._prepare_video_latents_stage( + request_inputs, + prompt_context, + device=device, + noise_scale=noise_scale, + image=None, + ) + + if image is not None: if isinstance(image, torch.Tensor): if image.ndim == 3: image = image.unsqueeze(0) elif isinstance(image, list) and image and isinstance(image[0], torch.Tensor): image = torch.stack(image, dim=0) + elif isinstance(image, PIL.Image.Image) or ( + isinstance(image, list) and image and isinstance(image[0], PIL.Image.Image) + ): + image = _preprocess_i2v_pil_images( + image, + height=request_inputs.height, + width=request_inputs.width, + device=device, + dtype=prompt_context.positive_connector_prompt_embeds.dtype, + ) else: image = self.video_processor.preprocess( image, @@ -509,32 +692,17 @@ class LTXI2VConditioningMixin: ) return self.prepare_latents( - image, - prompt_context.batch_size * request_inputs.num_videos_per_prompt, - self.transformer.config.in_channels, - request_inputs.height, - request_inputs.width, - request_inputs.num_frames, - noise_scale, - torch.float32, - device, - request_inputs.generator, - request_inputs.latents, - ) - - def _make_video_audio_scheduler( - self, - audio_scheduler: Any, - latent_num_frames: int, - latent_height: int, - latent_width: int, - ) -> Any: - return I2VVideoAudioScheduler( - self, - audio_scheduler, - latent_num_frames, - latent_height, - latent_width, + image=image, + batch_size=prompt_context.batch_size * request_inputs.num_videos_per_prompt, + num_channels_latents=self.transformer.config.in_channels, + height=request_inputs.height, + width=request_inputs.width, + num_frames=request_inputs.num_frames, + noise_scale=noise_scale, + dtype=prompt_context.positive_connector_prompt_embeds.dtype, + device=device, + generator=request_inputs.generator, + latents=request_inputs.latents, ) def _prepare_denoise_context_for_cfg( @@ -544,6 +712,8 @@ class LTXI2VConditioningMixin: ) -> LTXDenoiseContext: denoise_ctx = super()._prepare_denoise_context_for_cfg(forward_ctx, denoise_ctx) if denoise_ctx.conditioning_mask is None: + if self.unified_text_image_entry: + return denoise_ctx raise ValueError("LTX I2V denoising requires a conditioning mask.") mask_batch = denoise_ctx.conditioning_mask.shape[0] @@ -564,15 +734,46 @@ class LTXI2VConditioningMixin: ts: torch.Tensor, forward_ctx: LTXForwardContext, denoise_ctx: LTXDenoiseContext, + *, + video_token_count: int, + audio_token_count: int, ) -> dict[str, torch.Tensor]: - kwargs = super()._denoise_timestep_kwargs(ts, forward_ctx, denoise_ctx) + if denoise_ctx.conditioning_mask is None: + if self.unified_text_image_entry: + return super()._denoise_timestep_kwargs( + ts, + forward_ctx, + denoise_ctx, + video_token_count=video_token_count, + audio_token_count=audio_token_count, + ) + raise ValueError("LTX I2V denoising requires a conditioning mask.") + kwargs = super()._denoise_timestep_kwargs( + ts, + forward_ctx, + denoise_ctx, + video_token_count=video_token_count, + audio_token_count=audio_token_count, + ) conditioning_mask = ( denoise_ctx.conditioning_mask if forward_ctx.cfg_parallel_ready else denoise_ctx.conditioning_mask_for_model ) if conditioning_mask is None: raise ValueError("LTX I2V denoising requires a conditioning mask.") kwargs.update( - timestep=ts.unsqueeze(-1) * (1 - conditioning_mask), - audio_timestep=ts, + timestep=ts.reshape(-1, 1) * (1 - conditioning_mask), ) return kwargs + + def _video_guidance_model_sigma( + self, + sigma: torch.Tensor, + denoise_ctx: LTXDenoiseContext, + ) -> torch.Tensor: + if denoise_ctx.conditioning_mask is None: + if self.unified_text_image_entry: + return super()._video_guidance_model_sigma(sigma, denoise_ctx) + raise ValueError("LTX I2V guidance requires a conditioning mask.") + # The conditioned tokens are evaluated at timestep zero by the model, + # so velocity-to-x0 guidance must use that same per-token sigma. + return sigma.reshape(-1, 1, 1) * (1 - denoise_ctx.conditioning_mask).unsqueeze(-1) diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_denoise.py b/vllm_omni/diffusion/models/ltx2/ltx2_denoise.py index d0dfa3e54..262c9334c 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_denoise.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_denoise.py @@ -6,20 +6,22 @@ from __future__ import annotations import copy +import math from collections.abc import Callable, Iterable from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Protocol -import numpy as np import torch from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion import retrieve_timesteps from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch -from .ltx2_latents import LTXAVState +from .ltx2_guidance import euler_step_from_velocity +from .ltx2_latents import LTXAVState, unpack_audio_latents, unpad_audio_latents if TYPE_CHECKING: from .ltx2_conditioning import LTXPromptContext + from .ltx2_recipes import LTXPhaseRecipe from .ltx2_request import LTXRequestInputs @@ -41,7 +43,7 @@ class LTXForwardContext: padded_audio_num_frames: int timesteps: torch.Tensor audio_scheduler: Any - video_audio_scheduler: Any + video_audio_step_adapter: Any @property def batch_size(self) -> int: @@ -71,6 +73,7 @@ class LTXPhaseResult: forward_context: LTXForwardContext video: torch.Tensor audio: torch.Tensor + audio_for_next_phase: torch.Tensor | None = None class LTXDenoisePipeline(Protocol): @@ -122,61 +125,81 @@ def calculate_shift( return image_seq_len * slope + intercept -class VideoAudioScheduler: - """Composite scheduler dispatching video and audio updates.""" +class LTXVideoAudioStepAdapter: + """Expose the shared LTX Euler update through the distributed scheduler API.""" - def __init__(self, video_scheduler, audio_scheduler): - self.video_scheduler = video_scheduler - self.audio_scheduler = audio_scheduler - - def step(self, noise_pred, t, latents, return_dict=False, generator=None): - video_out = self.video_scheduler.step( - noise_pred[0], - t[0], - latents[0], - return_dict=False, - generator=generator, - )[0] - audio_out = self.audio_scheduler.step( - noise_pred[1], - t[1], - latents[1], - return_dict=False, - generator=generator, - )[0] - return ((video_out, audio_out),) - - -class I2VVideoAudioScheduler: - """Update the unconditioned video frames and the full audio state.""" - - def __init__(self, pipeline, audio_scheduler, latent_num_frames, latent_height, latent_width): - self.video_scheduler = pipeline.scheduler - self.audio_scheduler = audio_scheduler + def __init__( + self, + pipeline: Any, + audio_scheduler: Any, + latent_num_frames: int, + latent_height: int, + latent_width: int, + *, + image_conditioned: bool, + ) -> None: self._pipeline = pipeline + self._audio_scheduler = audio_scheduler self._latent_num_frames = latent_num_frames self._latent_height = latent_height self._latent_width = latent_width + self._image_conditioned = image_conditioned + self._step_index = 0 def step(self, noise_pred, t, latents, return_dict=False, generator=None): - video_out = self._pipeline._step_video_latents_i2v( - noise_pred[0], - latents[0], - t[0], - self._latent_num_frames, - self._latent_height, - self._latent_width, - ) - audio_out = self.audio_scheduler.step( - noise_pred[1], - t[1], + del t, return_dict, generator + if self._image_conditioned: + video_out = self._pipeline._step_video_latents_i2v( + noise_pred[0], + latents[0], + self._step_index, + self._latent_num_frames, + self._latent_height, + self._latent_width, + ) + else: + video_out = euler_step_from_velocity( + latents[0], + noise_pred[0], + self._pipeline.scheduler.sigmas, + self._step_index, + ) + audio_out = euler_step_from_velocity( latents[1], - return_dict=False, - generator=generator, - )[0] + noise_pred[1], + self._audio_scheduler.sigmas, + self._step_index, + ) + self._step_index += 1 return ((video_out, audio_out),) +def _official_ltx_sigmas(scheduler: Any, steps: int, device: torch.device) -> torch.Tensor: + """Build the official LTX one-stage sigma schedule in torch fp32.""" + config = scheduler.config + base_anchor = config.get("base_image_seq_len", 1024) + max_anchor = config.get("max_image_seq_len", 4096) + base_shift = config.get("base_shift", 0.95) + max_shift = config.get("max_shift", 2.05) + + sigmas = torch.linspace(1.0, 0.0, steps + 1) + slope = (max_shift - base_shift) / (max_anchor - base_anchor) + # Official LTX2 one-stage intentionally uses the max sequence anchor, so the shift stays at max_shift. + sigma_shift = max_anchor * slope + (base_shift - slope * base_anchor) + exp_shift = math.exp(sigma_shift) + sigmas = torch.where(sigmas != 0, exp_shift / (exp_shift + (1 / sigmas - 1)), 0) + + # Official non-distilled checkpoints explicitly set this to 0.1. Missing + # or None means the loaded scheduler disabled terminal stretching. + terminal = config.get("shift_terminal") + if terminal is not None: + non_zero = sigmas != 0 + one_minus_sigmas = 1.0 - sigmas[non_zero] + scale = one_minus_sigmas[-1] / (1.0 - terminal) + sigmas[non_zero] = 1.0 - one_minus_sigmas / scale + return sigmas.to(dtype=torch.float32, device=device) + + def _set_scheduler_sigmas(scheduler: Any, sigmas: torch.Tensor) -> torch.Tensor: sigmas = sigmas.to(torch.float32) timesteps = sigmas[:-1] * scheduler.config.get("num_train_timesteps", 1000) @@ -198,16 +221,20 @@ def prepare_scheduler_stage( latent_num_frames: int, latent_height: int, latent_width: int, + use_official_sigma_schedule: bool, + image_conditioned: bool = False, ) -> tuple[Any, Any, torch.Tensor]: if sigmas is not None and timesteps is not None: raise ValueError("Only one of `sigmas` or `timesteps` may be provided.") audio_scheduler = copy.deepcopy(pipeline.scheduler) - video_audio_scheduler = pipeline._make_video_audio_scheduler( + video_audio_step_adapter = LTXVideoAudioStepAdapter( + pipeline, audio_scheduler, latent_num_frames, latent_height, latent_width, + image_conditioned=image_conditioned, ) if sigmas is not None: scheduler_sigmas = torch.as_tensor(sigmas, dtype=torch.float32, device=device) @@ -217,12 +244,14 @@ def prepare_scheduler_stage( scheduler_sigmas = torch.cat([scheduler_sigmas, scheduler_sigmas.new_zeros(1)]) timesteps_tensor = _set_scheduler_sigmas(pipeline.scheduler, scheduler_sigmas) _set_scheduler_sigmas(audio_scheduler, scheduler_sigmas.clone()) - return audio_scheduler, video_audio_scheduler, timesteps_tensor + return audio_scheduler, video_audio_step_adapter, timesteps_tensor + + if sigmas is None and timesteps is None and use_official_sigma_schedule: + scheduler_sigmas = _official_ltx_sigmas(pipeline.scheduler, request_inputs.num_inference_steps, device) + timesteps_tensor = _set_scheduler_sigmas(pipeline.scheduler, scheduler_sigmas) + _set_scheduler_sigmas(audio_scheduler, scheduler_sigmas.clone()) + return audio_scheduler, video_audio_step_adapter, timesteps_tensor - sigmas = np.linspace(1.0, 1 / request_inputs.num_inference_steps, request_inputs.num_inference_steps) - # Official LTX2 one-stage omits the latent, so mu stays at max_shift across resolutions. - # Mu varies with latent token count only when the scheduler receives a latent, as in official HQ two-stage; - # explicit sigma schedules bypass mu entirely. mu = calculate_shift( pipeline.scheduler.config.get("max_image_seq_len", 4096), pipeline.scheduler.config.get("base_image_seq_len", 1024), @@ -246,7 +275,7 @@ def prepare_scheduler_stage( sigmas=sigmas, mu=mu, ) - return audio_scheduler, video_audio_scheduler, timesteps_tensor + return audio_scheduler, video_audio_step_adapter, timesteps_tensor def prepare_rope_coords_stage( @@ -283,15 +312,25 @@ def build_transformer_kwargs( encoder_attention_mask: torch.Tensor | None, audio_encoder_attention_mask: torch.Tensor | None, ts: torch.Tensor, + attention_kwargs: dict[str, Any] | None = None, ) -> dict[str, Any]: + del encoder_attention_mask, audio_encoder_attention_mask return { "hidden_states": hidden_states, "audio_hidden_states": audio_hidden_states, "encoder_hidden_states": encoder_hidden_states, "audio_encoder_hidden_states": audio_encoder_hidden_states, - **pipeline._denoise_timestep_kwargs(ts, forward_ctx, denoise_ctx), - "encoder_attention_mask": encoder_attention_mask, - "audio_encoder_attention_mask": audio_encoder_attention_mask, + **pipeline._denoise_timestep_kwargs( + ts, + forward_ctx, + denoise_ctx, + video_token_count=hidden_states.shape[1], + audio_token_count=audio_hidden_states.shape[1], + ), + # This is valid only because LTX connectors replace every padding token + # with a learned register, making all output context tokens valid. + "encoder_attention_mask": None, + "audio_encoder_attention_mask": None, "num_frames": forward_ctx.latent_num_frames, "height": forward_ctx.latent_height, "width": forward_ctx.latent_width, @@ -299,7 +338,7 @@ def build_transformer_kwargs( "audio_num_frames": forward_ctx.padded_audio_num_frames, "video_coords": denoise_ctx.video_coords, "audio_coords": denoise_ctx.audio_coords, - "attention_kwargs": forward_ctx.attention_kwargs, + "attention_kwargs": forward_ctx.attention_kwargs if attention_kwargs is None else attention_kwargs, "return_dict": False, } @@ -317,7 +356,7 @@ def step_denoised_latents( (timestep, timestep), (denoise_ctx.latents, denoise_ctx.audio_latents), do_true_cfg=pipeline.do_classifier_free_guidance, - per_request_scheduler=forward_ctx.video_audio_scheduler, + per_request_scheduler=forward_ctx.video_audio_step_adapter, ) return pipeline._synchronize_cfg_parallel_step_output( latents, @@ -338,11 +377,12 @@ class LTXPhaseExecutor: sigmas: list[float] | None, timesteps: list[int] | None, attention_kwargs: dict[str, Any] | None, + phase_recipe: LTXPhaseRecipe, image: Any | None = None, prompt_context: LTXPromptContext | None = None, ) -> LTXPhaseResult: pipeline._check_forward_inputs(request_inputs, image=image) - cfg_parallel_ready = pipeline._setup_forward_runtime(request_inputs, attention_kwargs) + cfg_parallel_ready = pipeline._setup_forward_runtime(req, request_inputs, attention_kwargs) device = pipeline.device if prompt_context is None: prompt_context = pipeline._prepare_prompt_context( @@ -372,7 +412,7 @@ class LTXPhaseExecutor: noise_scale=noise_scale, ) ) - audio_scheduler, video_audio_scheduler, timesteps_tensor = prepare_scheduler_stage( + audio_scheduler, video_audio_step_adapter, timesteps_tensor = prepare_scheduler_stage( pipeline, request_inputs, device=device, @@ -381,6 +421,8 @@ class LTXPhaseExecutor: latent_num_frames=latent_num_frames, latent_height=latent_height, latent_width=latent_width, + use_official_sigma_schedule=phase_recipe.use_official_sigma_schedule, + image_conditioned=conditioning_mask is not None, ) forward_ctx = LTXForwardContext( req=req, @@ -397,7 +439,7 @@ class LTXPhaseExecutor: padded_audio_num_frames=padded_audio_num_frames, timesteps=timesteps_tensor, audio_scheduler=audio_scheduler, - video_audio_scheduler=video_audio_scheduler, + video_audio_step_adapter=video_audio_step_adapter, ) video_coords, audio_coords = prepare_rope_coords_stage(pipeline, forward_ctx, latents, audio_latents) denoise_ctx = LTXDenoiseContext( @@ -427,4 +469,13 @@ class LTXPhaseExecutor: state.video, state.audio, ) - return LTXPhaseResult(forward_context=forward_ctx, video=latents, audio=audio_latents) + normalized_audio = unpack_audio_latents( + unpad_audio_latents(state.audio, forward_ctx.original_audio_num_frames), + num_mel_bins=forward_ctx.latent_mel_bins, + ) + return LTXPhaseResult( + forward_context=forward_ctx, + video=latents, + audio=audio_latents, + audio_for_next_phase=normalized_audio, + ) diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_guidance.py b/vllm_omni/diffusion/models/ltx2/ltx2_guidance.py index 395f8e7af..0b3d2dc2b 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_guidance.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_guidance.py @@ -1,16 +1,16 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Guidance strategies for the LTX model family.""" +"""Official multi-modal guidance for the LTX model family.""" from __future__ import annotations -from typing import TYPE_CHECKING, Any, Protocol +import math +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any import torch -from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion import rescale_noise_cfg -from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin from vllm_omni.diffusion.distributed.parallel_state import ( get_cfg_group, get_classifier_free_guidance_rank, @@ -22,358 +22,409 @@ if TYPE_CHECKING: from .ltx2_latents import LTXAVState -class LTXGuidanceStrategy(Protocol): - """Model-version guidance behavior consumed by the shared pipeline.""" +@dataclass(frozen=True) +class LTXModalityGuidance: + """Guidance parameters applied to one LTX output modality.""" - def validate_cfg_world_size(self, cfg_world_size: int) -> None: ... + cfg_scale: float = 1.0 + stg_scale: float = 0.0 + modality_scale: float = 1.0 + rescale_scale: float = 0.0 + stg_blocks: tuple[int, ...] = () - def prepare_denoise_context( - self, - pipeline: Any, - forward_ctx: LTXForwardContext, - denoise_ctx: LTXDenoiseContext, - ) -> LTXDenoiseContext: ... + @property + def do_cfg(self) -> bool: + return not math.isclose(self.cfg_scale, 1.0) - def timestep_kwargs( - self, - ts: torch.Tensor, - forward_ctx: LTXForwardContext, - denoise_ctx: LTXDenoiseContext, - ) -> dict[str, torch.Tensor]: ... + @property + def do_stg(self) -> bool: + return not math.isclose(self.stg_scale, 0.0) and bool(self.stg_blocks) - def combine_cfg_noise( - self, - pipeline: Any, - positive_noise_pred: tuple[torch.Tensor, torch.Tensor], - negative_noise_pred: tuple[torch.Tensor, torch.Tensor], - true_cfg_scale: float, - cfg_normalize: bool, - context: dict[str, Any], - ) -> tuple[torch.Tensor, torch.Tensor]: ... - - def predict_noise( - self, - pipeline: Any, - index: int, - timestep: torch.Tensor, - state: LTXAVState, - forward_ctx: LTXForwardContext, - denoise_ctx: LTXDenoiseContext, - ) -> tuple[torch.Tensor, torch.Tensor]: ... + @property + def do_modality_guidance(self) -> bool: + return not math.isclose(self.modality_scale, 1.0) -class _LTXGuidanceStrategyBase: - def validate_cfg_world_size(self, cfg_world_size: int) -> None: - del cfg_world_size +@dataclass(frozen=True) +class LTXGuidanceSpec: + """Video and audio guidance requested for one denoise phase.""" - def prepare_denoise_context( - self, - pipeline: Any, - forward_ctx: LTXForwardContext, - denoise_ctx: LTXDenoiseContext, - ) -> LTXDenoiseContext: - del pipeline, forward_ctx - return denoise_ctx + video: LTXModalityGuidance = LTXModalityGuidance() + audio: LTXModalityGuidance = LTXModalityGuidance() - def timestep_kwargs( - self, - ts: torch.Tensor, - forward_ctx: LTXForwardContext, - denoise_ctx: LTXDenoiseContext, - ) -> dict[str, torch.Tensor]: - del forward_ctx, denoise_ctx - return {"timestep": ts} + @classmethod + def positive_only(cls) -> LTXGuidanceSpec: + return cls() + + @property + def do_cfg(self) -> bool: + return self.video.do_cfg or self.audio.do_cfg + + @property + def do_stg(self) -> bool: + return self.video.do_stg or self.audio.do_stg + + @property + def do_modality_guidance(self) -> bool: + return self.video.do_modality_guidance or self.audio.do_modality_guidance + + @property + def do_rescale(self) -> bool: + return self.video.rescale_scale != 0.0 or self.audio.rescale_scale != 0.0 -def combine_velocity_via_x0( +@dataclass(frozen=True) +class LTXDenoisePass: + """One conditioned Transformer evaluation in a guidance batch.""" + + name: str + negative_video_context: bool = False + negative_audio_context: bool = False + + +@dataclass(frozen=True) +class LTXGuidancePlan: + """Concrete Transformer passes derived from a guidance specification.""" + + spec: LTXGuidanceSpec + passes: tuple[LTXDenoisePass, ...] + + @classmethod + def build(cls, spec: LTXGuidanceSpec) -> LTXGuidancePlan: + passes = [LTXDenoisePass("cond")] + if spec.do_cfg: + passes.append( + LTXDenoisePass( + "uncond", + negative_video_context=spec.video.do_cfg, + negative_audio_context=spec.audio.do_cfg, + ) + ) + if spec.do_stg: + passes.append(LTXDenoisePass("ptb")) + if spec.do_modality_guidance: + passes.append(LTXDenoisePass("mod")) + return cls(spec=spec, passes=tuple(passes)) + + @property + def names(self) -> tuple[str, ...]: + return tuple(item.name for item in self.passes) + + @property + def cfg_parallel_compatible(self) -> bool: + return self.names == ("cond", "uncond") and not self.spec.do_rescale + + +def x0_from_velocity(sample: torch.Tensor, velocity: torch.Tensor, sigma: torch.Tensor) -> torch.Tensor: + """Convert velocity to x0 using the official fp32 arithmetic order.""" + sigma = sigma.to(torch.float32) + return (sample.to(torch.float32) - velocity.to(torch.float32) * sigma).to(sample.dtype) + + +def velocity_from_x0(sample: torch.Tensor, x0: torch.Tensor, sigma: torch.Tensor) -> torch.Tensor: + """Convert x0 back to velocity using the official fp32 arithmetic order.""" + sigma = sigma.to(torch.float32) + return ((sample.to(torch.float32) - x0.to(torch.float32)) / sigma).to(sample.dtype) + + +def euler_step_from_velocity( sample: torch.Tensor, - positive_velocity: torch.Tensor, - negative_velocity: torch.Tensor, - sigma: torch.Tensor, - guidance_scale: float, + velocity: torch.Tensor, + sigmas: torch.Tensor, + step_index: int, ) -> torch.Tensor: - """Apply CFG in x0 space and convert the result back to velocity.""" - x0_cond = sample - positive_velocity * sigma - x0_uncond = sample - negative_velocity * sigma - x0_guided = x0_cond + (guidance_scale - 1) * (x0_cond - x0_uncond) - return (sample - x0_guided) / sigma + sigma = sigmas[step_index].to(torch.float32) + sigma_next = sigmas[step_index + 1].to(torch.float32) + return (sample.float() + velocity.float() * (sigma_next - sigma)).to(sample.dtype) -class LTXLegacyVelocityGuidance(_LTXGuidanceStrategyBase): - """Existing LTX2 Diffusers-style velocity-space CFG behavior.""" - - def combine_cfg_noise( - self, - pipeline: Any, - positive_noise_pred: tuple[torch.Tensor, torch.Tensor], - negative_noise_pred: tuple[torch.Tensor, torch.Tensor], - true_cfg_scale: float, - cfg_normalize: bool, - context: dict[str, Any], - ) -> tuple[torch.Tensor, torch.Tensor]: - del context - video_pos, audio_pos = positive_noise_pred - video_neg, audio_neg = negative_noise_pred - video_combined = CFGParallelMixin.combine_cfg_noise( - pipeline, - video_pos, - video_neg, - true_cfg_scale, - cfg_normalize, - ) - audio_combined = CFGParallelMixin.combine_cfg_noise( - pipeline, - audio_pos, - audio_neg, - true_cfg_scale, - cfg_normalize, - ) - if pipeline.guidance_rescale and pipeline.guidance_rescale > 0: - video_combined = rescale_noise_cfg( - video_combined, - video_pos, - guidance_rescale=pipeline.guidance_rescale, - ) - audio_combined = rescale_noise_cfg( - audio_combined, - audio_pos, - guidance_rescale=pipeline.guidance_rescale, - ) - return video_combined, audio_combined - - def predict_noise( - self, - pipeline: Any, - index: int, - timestep: torch.Tensor, - state: LTXAVState, - forward_ctx: LTXForwardContext, - denoise_ctx: LTXDenoiseContext, - ) -> tuple[torch.Tensor, torch.Tensor]: - del index - prompt_context = forward_ctx.prompt_context - video_input = state.video.to(prompt_context.positive_connector_prompt_embeds.dtype) - audio_input = state.audio.to(prompt_context.positive_connector_prompt_embeds.dtype) - expanded_timestep = timestep.expand(video_input.shape[0]) - positive_kwargs = pipeline._build_transformer_kwargs( - forward_ctx, - denoise_ctx, - hidden_states=video_input, - audio_hidden_states=audio_input, - encoder_hidden_states=prompt_context.positive_connector_prompt_embeds, - audio_encoder_hidden_states=prompt_context.positive_connector_audio_prompt_embeds, - encoder_attention_mask=prompt_context.positive_connector_attention_mask, - audio_encoder_attention_mask=prompt_context.positive_connector_attention_mask, - ts=expanded_timestep, - ) - negative_kwargs = ( - { - **positive_kwargs, - "encoder_hidden_states": prompt_context.negative_connector_prompt_embeds, - "audio_encoder_hidden_states": prompt_context.negative_connector_audio_prompt_embeds, - "encoder_attention_mask": prompt_context.negative_connector_attention_mask, - "audio_encoder_attention_mask": prompt_context.negative_connector_attention_mask, - } - if pipeline.do_classifier_free_guidance - else None - ) - return pipeline.predict_noise_maybe_with_cfg( - do_true_cfg=pipeline.do_classifier_free_guidance, - true_cfg_scale=forward_ctx.request_inputs.guidance_scale, - positive_kwargs=positive_kwargs, - negative_kwargs=negative_kwargs, - cfg_normalize=False, - ) +def combine_guided_x0( + *, + cond: torch.Tensor, + uncond_text: torch.Tensor | float, + uncond_perturbed: torch.Tensor | float, + uncond_modality: torch.Tensor | float, + guidance: LTXModalityGuidance, +) -> torch.Tensor: + dtype = cond.dtype + cond = cond.float() + uncond_text = uncond_text.float() if isinstance(uncond_text, torch.Tensor) else uncond_text + uncond_perturbed = uncond_perturbed.float() if isinstance(uncond_perturbed, torch.Tensor) else uncond_perturbed + uncond_modality = uncond_modality.float() if isinstance(uncond_modality, torch.Tensor) else uncond_modality + pred = cond + (guidance.cfg_scale - 1) * (cond - uncond_text) + if guidance.do_stg: + pred = pred + guidance.stg_scale * (cond - uncond_perturbed) + pred = pred + (guidance.modality_scale - 1) * (cond - uncond_modality) + if guidance.rescale_scale != 0.0: + # The pinned official one-stage entry runs one generated sample and + # uses its full-tensor standard deviation. In Omni, dimension 0 may + # instead contain independently scheduled requests. Reduce each item + # separately so its result cannot depend on unrelated co-batched work. + reduce_dims = tuple(range(1, pred.ndim)) + cond_std = cond.std(dim=reduce_dims, keepdim=True) + pred_std = pred.std(dim=reduce_dims, keepdim=True) + eps = torch.finfo(pred.dtype).eps + factor = cond_std / pred_std.clamp_min(eps) + factor = torch.where(pred_std > eps, factor, torch.ones_like(factor)) + factor = guidance.rescale_scale * factor + (1 - guidance.rescale_scale) + pred = pred * factor + return pred.to(dtype) -class LTXOfficialX0Guidance(_LTXGuidanceStrategyBase): - """Official LTX x0-space CFG behavior currently used by LTX2.3.""" +def _repeat_batch(tensor: torch.Tensor, repeats: int) -> torch.Tensor: + return tensor.repeat((repeats,) + (1,) * (tensor.ndim - 1)) - def validate_cfg_world_size(self, cfg_world_size: int) -> None: - if cfg_world_size not in (1, 2): - raise ValueError( - f"LTX x0 guidance supports CFG parallelism with cfg_parallel_size 1 or 2, but got {cfg_world_size}." - ) +def _mask_for_pass( + pass_count: int, + batch_size: int, + pass_index: int, + reference: torch.Tensor, +) -> torch.Tensor: + mask = reference.new_ones((pass_count * batch_size, 1, 1)) + start = pass_index * batch_size + mask[start : start + batch_size] = 0 + return mask + + +def build_perturbation_kwargs( + plan: LTXGuidancePlan, + batch_size: int, + reference: torch.Tensor, +) -> dict[str, Any]: + kwargs: dict[str, Any] = {} + if "ptb" in plan.names: + mask = _mask_for_pass(len(plan.passes), batch_size, plan.names.index("ptb"), reference) + if plan.spec.video.do_stg: + kwargs["video_self_attention_mask"] = mask + kwargs["video_self_attention_blocks"] = plan.spec.video.stg_blocks + if plan.spec.audio.do_stg: + kwargs["audio_self_attention_mask"] = mask + kwargs["audio_self_attention_blocks"] = plan.spec.audio.stg_blocks + if "mod" in plan.names: + mask = _mask_for_pass(len(plan.passes), batch_size, plan.names.index("mod"), reference) + kwargs["a2v_cross_attention_mask"] = mask + kwargs["v2a_cross_attention_mask"] = mask + return kwargs + + +class LTXGuidanceExecutor: + """Build and execute official LTX multi-guidance Transformer passes.""" + + @staticmethod + def validate_cfg_world_size(plan: LTXGuidancePlan, cfg_world_size: int) -> None: + if cfg_world_size == 1: + return + if cfg_world_size != 2: + raise ValueError(f"LTX CFG parallelism supports cfg_parallel_size 1 or 2, got {cfg_world_size}.") + if not plan.cfg_parallel_compatible: + raise ValueError("LTX CFG parallelism only supports CFG-only guidance without rescale.") + + @staticmethod def prepare_denoise_context( - self, - pipeline: Any, - forward_ctx: LTXForwardContext, + plan: LTXGuidancePlan, + cfg_parallel_ready: bool, denoise_ctx: LTXDenoiseContext, ) -> LTXDenoiseContext: - if pipeline.do_classifier_free_guidance and not forward_ctx.cfg_parallel_ready: - denoise_ctx.video_coords = denoise_ctx.video_coords.repeat( - (2,) + (1,) * (denoise_ctx.video_coords.ndim - 1) - ) - denoise_ctx.audio_coords = denoise_ctx.audio_coords.repeat( - (2,) + (1,) * (denoise_ctx.audio_coords.ndim - 1) - ) + if len(plan.passes) > 1 and not cfg_parallel_ready: + denoise_ctx.video_coords = _repeat_batch(denoise_ctx.video_coords, len(plan.passes)) + denoise_ctx.audio_coords = _repeat_batch(denoise_ctx.audio_coords, len(plan.passes)) return denoise_ctx + @staticmethod def timestep_kwargs( - self, ts: torch.Tensor, - forward_ctx: LTXForwardContext, - denoise_ctx: LTXDenoiseContext, + video_token_count: int, + audio_token_count: int, ) -> dict[str, torch.Tensor]: - del forward_ctx, denoise_ctx - return {"timestep": ts, "sigma": ts} + return { + "timestep": ts.reshape(-1, 1).expand(-1, video_token_count), + "audio_timestep": ts.reshape(-1, 1).expand(-1, audio_token_count), + "sigma": ts, + "audio_sigma": ts, + } - def combine_cfg_noise( - self, - pipeline: Any, - positive_noise_pred: tuple[torch.Tensor, torch.Tensor], - negative_noise_pred: tuple[torch.Tensor, torch.Tensor], - true_cfg_scale: float, - cfg_normalize: bool, - context: dict[str, Any], - ) -> tuple[torch.Tensor, torch.Tensor]: - required = ("video_latents", "audio_latents", "video_sigma", "audio_sigma") - if any(context.get(name) is None for name in required): - raise ValueError("LTX x0-space CFG requires video/audio latents and sigmas.") - - video_pos, audio_pos = positive_noise_pred - video_neg, audio_neg = negative_noise_pred - video_combined = combine_velocity_via_x0( - context["video_latents"], - video_pos, - video_neg, - context["video_sigma"], - true_cfg_scale, + @staticmethod + def combine_cfg_velocity( + sample: torch.Tensor, + positive_velocity: torch.Tensor, + negative_velocity: torch.Tensor, + sigma: torch.Tensor, + guidance_scale: float, + *, + model_sigma: torch.Tensor | None = None, + ) -> torch.Tensor: + guidance = LTXModalityGuidance(cfg_scale=guidance_scale) + model_sigma = sigma if model_sigma is None else model_sigma + guided_x0 = combine_guided_x0( + cond=x0_from_velocity(sample, positive_velocity, model_sigma), + uncond_text=x0_from_velocity(sample, negative_velocity, model_sigma), + uncond_perturbed=0.0, + uncond_modality=0.0, + guidance=guidance, ) - audio_combined = combine_velocity_via_x0( - context["audio_latents"], - audio_pos, - audio_neg, - context["audio_sigma"], - true_cfg_scale, - ) - if cfg_normalize: - video_combined = pipeline.cfg_normalize_function(video_pos, video_combined) - audio_combined = pipeline.cfg_normalize_function(audio_pos, audio_combined) - return video_combined, audio_combined + return velocity_from_x0(sample, guided_x0, sigma) def predict_parallel_cfg( self, pipeline: Any, - true_cfg_scale: float, - positive_kwargs: dict[str, Any], - negative_kwargs: dict[str, Any], - cfg_normalize: bool = True, - output_slice: int | None = None, - **context: Any, - ) -> tuple[torch.Tensor, torch.Tensor]: - def maybe_slice(pred: tuple[torch.Tensor, torch.Tensor]) -> tuple[torch.Tensor, torch.Tensor]: - if output_slice is None: - return pred - return pred[0][:, :output_slice], pred[1][:, :output_slice] - - cfg_world_size = get_classifier_free_guidance_world_size() - self.validate_cfg_world_size(cfg_world_size) - if cfg_world_size != 2: - raise ValueError(f"Parallel LTX x0 guidance requires cfg_parallel_size 2, but got {cfg_world_size}.") - - cfg_rank = get_classifier_free_guidance_rank() - branch_kwargs = positive_kwargs if cfg_rank == 0 else negative_kwargs - local_video_pred, local_audio_pred = maybe_slice(pipeline.predict_noise(**branch_kwargs)) - cfg_group = get_cfg_group() - gathered_video = cfg_group.all_gather(local_video_pred, separate_tensors=True) - gathered_audio = cfg_group.all_gather(local_audio_pred, separate_tensors=True) - return self.combine_cfg_noise( - pipeline, - (gathered_video[0], gathered_audio[0]), - (gathered_video[1], gathered_audio[1]), - true_cfg_scale, - cfg_normalize, - context, - ) - - def predict_noise( - self, - pipeline: Any, + plan: LTXGuidancePlan, index: int, timestep: torch.Tensor, state: LTXAVState, forward_ctx: LTXForwardContext, denoise_ctx: LTXDenoiseContext, ) -> tuple[torch.Tensor, torch.Tensor]: - prompt_context = forward_ctx.prompt_context - guidance_scale = forward_ctx.request_inputs.guidance_scale - audio_scheduler = forward_ctx.audio_scheduler - if forward_ctx.cfg_parallel_ready: - video_input = state.video.to(prompt_context.positive_connector_prompt_embeds.dtype) - audio_input = state.audio.to(prompt_context.positive_connector_prompt_embeds.dtype) - ts = timestep.expand(video_input.shape[0]) - positive_kwargs = pipeline._build_transformer_kwargs( - forward_ctx, - denoise_ctx, - hidden_states=video_input, - audio_hidden_states=audio_input, - encoder_hidden_states=prompt_context.positive_connector_prompt_embeds, - audio_encoder_hidden_states=prompt_context.positive_connector_audio_prompt_embeds, - encoder_attention_mask=prompt_context.positive_connector_attention_mask, - audio_encoder_attention_mask=prompt_context.positive_connector_attention_mask, - ts=ts, - ) - negative_kwargs = { - **positive_kwargs, - "encoder_hidden_states": prompt_context.negative_connector_prompt_embeds, - "audio_encoder_hidden_states": prompt_context.negative_connector_audio_prompt_embeds, - "encoder_attention_mask": prompt_context.negative_connector_attention_mask, - "audio_encoder_attention_mask": prompt_context.negative_connector_attention_mask, - } - return self.predict_parallel_cfg( - pipeline, - true_cfg_scale=guidance_scale, - positive_kwargs=positive_kwargs, - negative_kwargs=negative_kwargs, - cfg_normalize=False, - video_latents=state.video, - audio_latents=state.audio, - video_sigma=pipeline.scheduler.sigmas[index], - audio_sigma=audio_scheduler.sigmas[index], - ) - - video_input = torch.cat([state.video] * 2) if pipeline.do_classifier_free_guidance else state.video - video_input = video_input.to(prompt_context.connector_prompt_embeds.dtype) - audio_input = torch.cat([state.audio] * 2) if pipeline.do_classifier_free_guidance else state.audio - audio_input = audio_input.to(prompt_context.connector_prompt_embeds.dtype) - ts = timestep.expand(video_input.shape[0]) - transformer_kwargs = pipeline._build_transformer_kwargs( + self.validate_cfg_world_size(plan, get_classifier_free_guidance_world_size()) + prompt = forward_ctx.prompt_context + negative = get_classifier_free_guidance_rank() == 1 + video_context = ( + prompt.negative_connector_prompt_embeds + if negative and plan.spec.video.do_cfg + else prompt.positive_connector_prompt_embeds + ) + audio_context = ( + prompt.negative_connector_audio_prompt_embeds + if negative and plan.spec.audio.do_cfg + else prompt.positive_connector_audio_prompt_embeds + ) + if video_context is None or audio_context is None: + raise ValueError("Negative prompt context is required when LTX CFG is enabled.") + video_input = state.video.to(prompt.positive_connector_prompt_embeds.dtype) + audio_input = state.audio.to(prompt.positive_connector_audio_prompt_embeds.dtype) + kwargs = pipeline._build_transformer_kwargs( forward_ctx, denoise_ctx, hidden_states=video_input, audio_hidden_states=audio_input, - encoder_hidden_states=prompt_context.connector_prompt_embeds, - audio_encoder_hidden_states=prompt_context.connector_audio_prompt_embeds, - encoder_attention_mask=prompt_context.connector_attention_mask, - audio_encoder_attention_mask=prompt_context.connector_attention_mask, - ts=ts, + encoder_hidden_states=video_context, + audio_encoder_hidden_states=audio_context, + encoder_attention_mask=None, + audio_encoder_attention_mask=None, + ts=timestep.expand(video_input.shape[0]), ) - with pipeline._transformer_cache_context("cond_uncond"): - video_pred, audio_pred = pipeline.transformer(**transformer_kwargs) - video_pred = video_pred.float() - audio_pred = audio_pred.float() - - if pipeline.do_classifier_free_guidance: - video_uncond, video_cond = video_pred.chunk(2) - video_pred = combine_velocity_via_x0( + local_video, local_audio = pipeline.predict_noise(**kwargs) + group = get_cfg_group() + video = group.all_gather(local_video, separate_tensors=True) + audio = group.all_gather(local_audio, separate_tensors=True) + video_sigma = pipeline.scheduler.sigmas[index] + return ( + self.combine_cfg_velocity( state.video, - video_cond, - video_uncond, - pipeline.scheduler.sigmas[index], - guidance_scale, - ) - audio_uncond, audio_cond = audio_pred.chunk(2) - audio_pred = combine_velocity_via_x0( + video[0], + video[1], + video_sigma, + plan.spec.video.cfg_scale, + model_sigma=pipeline._video_guidance_model_sigma(video_sigma, denoise_ctx), + ), + self.combine_cfg_velocity( state.audio, - audio_cond, - audio_uncond, - audio_scheduler.sigmas[index], - guidance_scale, + audio[0], + audio[1], + forward_ctx.audio_scheduler.sigmas[index], + plan.spec.audio.cfg_scale, + ), + ) + + def predict_noise( + self, + pipeline: Any, + plan: LTXGuidancePlan, + index: int, + timestep: torch.Tensor, + state: LTXAVState, + forward_ctx: LTXForwardContext, + denoise_ctx: LTXDenoiseContext, + ) -> tuple[torch.Tensor, torch.Tensor]: + if forward_ctx.cfg_parallel_ready: + return self.predict_parallel_cfg(pipeline, plan, index, timestep, state, forward_ctx, denoise_ctx) + + prompt = forward_ctx.prompt_context + video_contexts: list[torch.Tensor] = [] + audio_contexts: list[torch.Tensor] = [] + for denoise_pass in plan.passes: + video_context = ( + prompt.negative_connector_prompt_embeds + if denoise_pass.negative_video_context + else prompt.positive_connector_prompt_embeds ) - return video_pred, audio_pred + audio_context = ( + prompt.negative_connector_audio_prompt_embeds + if denoise_pass.negative_audio_context + else prompt.positive_connector_audio_prompt_embeds + ) + if video_context is None or audio_context is None: + raise ValueError("Negative prompt context is required when LTX CFG is enabled.") + video_contexts.append(video_context) + audio_contexts.append(audio_context) + + pass_count = len(plan.passes) + video_input = _repeat_batch(state.video, pass_count).to(prompt.positive_connector_prompt_embeds.dtype) + audio_input = _repeat_batch(state.audio, pass_count).to(prompt.positive_connector_audio_prompt_embeds.dtype) + attention_kwargs = dict(forward_ctx.attention_kwargs or {}) + perturbations = build_perturbation_kwargs(plan, state.video.shape[0], video_input) + if perturbations: + attention_kwargs["ltx_perturbation_kwargs"] = perturbations + kwargs = pipeline._build_transformer_kwargs( + forward_ctx, + denoise_ctx, + hidden_states=video_input, + audio_hidden_states=audio_input, + encoder_hidden_states=torch.cat(video_contexts), + audio_encoder_hidden_states=torch.cat(audio_contexts), + encoder_attention_mask=None, + audio_encoder_attention_mask=None, + ts=timestep.expand(video_input.shape[0]), + attention_kwargs=attention_kwargs, + ) + with pipeline._transformer_cache_context("guided"): + video_velocity, audio_velocity = pipeline.transformer(**kwargs) + video_splits = dict(zip(plan.names, video_velocity.chunk(pass_count), strict=True)) + audio_splits = dict(zip(plan.names, audio_velocity.chunk(pass_count), strict=True)) + + def guide_modality( + sample: torch.Tensor, + splits: dict[str, torch.Tensor], + sigma: torch.Tensor, + guidance: LTXModalityGuidance, + *, + model_sigma: torch.Tensor | None = None, + ) -> torch.Tensor: + model_sigma = sigma if model_sigma is None else model_sigma + # Official guidance reduces contiguous BSC tensors. The fused + # transformer output may be channel-major after splitting, which + # changes fp32 reduction order and can cross bf16 rounding bounds. + x0 = {name: x0_from_velocity(sample, value, model_sigma).contiguous() for name, value in splits.items()} + guided = combine_guided_x0( + cond=x0["cond"], + uncond_text=x0.get("uncond", 0.0), + uncond_perturbed=x0.get("ptb", 0.0), + uncond_modality=x0.get("mod", 0.0), + guidance=guidance, + ) + return velocity_from_x0(sample, guided, sigma) + + return ( + guide_modality( + state.video, + video_splits, + pipeline.scheduler.sigmas[index], + plan.spec.video, + model_sigma=pipeline._video_guidance_model_sigma( + pipeline.scheduler.sigmas[index], + denoise_ctx, + ), + ), + guide_modality( + state.audio, + audio_splits, + forward_ctx.audio_scheduler.sigmas[index], + plan.spec.audio, + ), + ) -LTX_LEGACY_VELOCITY_GUIDANCE = LTXLegacyVelocityGuidance() -LTX_OFFICIAL_X0_GUIDANCE = LTXOfficialX0Guidance() +LTX_GUIDANCE_EXECUTOR = LTXGuidanceExecutor() + +# Kept as a narrow compatibility helper for CFG-parallel tests and extensions. +combine_velocity_via_x0 = LTXGuidanceExecutor.combine_cfg_velocity diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_latents.py b/vllm_omni/diffusion/models/ltx2/ltx2_latents.py index 4da3edf92..bb0155e79 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_latents.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_latents.py @@ -120,45 +120,14 @@ def create_noised_state( return noise_scale * noise + (1 - noise_scale) * latents -def pack_audio_latents( - latents: torch.Tensor, - patch_size: int | None = None, - patch_size_t: int | None = None, -) -> torch.Tensor: - if patch_size is not None and patch_size_t is not None: - batch_size, _, latent_length, latent_mel_bins = latents.shape - post_patch_latent_length = latent_length / patch_size_t - post_patch_mel_bins = latent_mel_bins / patch_size - latents = latents.reshape( - batch_size, - -1, - post_patch_latent_length, - patch_size_t, - post_patch_mel_bins, - patch_size, - ) - return latents.permute(0, 2, 4, 1, 3, 5).flatten(3, 5).flatten(1, 2) +def pack_audio_latents(latents: torch.Tensor) -> torch.Tensor: return latents.transpose(1, 2).flatten(2, 3) def unpack_audio_latents( latents: torch.Tensor, - latent_length: int, num_mel_bins: int, - patch_size: int | None = None, - patch_size_t: int | None = None, ) -> torch.Tensor: - if patch_size is not None and patch_size_t is not None: - batch_size = latents.size(0) - latents = latents.reshape( - batch_size, - latent_length, - num_mel_bins, - -1, - patch_size_t, - patch_size, - ) - return latents.permute(0, 3, 1, 4, 2, 5).flatten(4, 5).flatten(2, 3) return latents.unflatten(2, (-1, num_mel_bins)).transpose(1, 2) @@ -224,18 +193,19 @@ def prepare_video_latents( vae_spatial_compression_ratio=pipeline.vae_spatial_compression_ratio, vae_temporal_compression_ratio=pipeline.vae_temporal_compression_ratio, ) - shape = (batch_size, num_channels_latents, num_frames, height, width) + spatial_patch_size = pipeline.transformer_spatial_patch_size + temporal_patch_size = pipeline.transformer_temporal_patch_size + shape = ( + batch_size, + (num_frames // temporal_patch_size) * (height // spatial_patch_size) * (width // spatial_patch_size), + num_channels_latents * temporal_patch_size * spatial_patch_size * spatial_patch_size, + ) if isinstance(generator, list) and len(generator) != batch_size: raise ValueError( f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" f" size of {batch_size}. Make sure the batch size matches the length of the generators." ) - latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) - return pack_latents( - latents, - pipeline.transformer_spatial_patch_size, - pipeline.transformer_temporal_patch_size, - ) + return randn_tensor(shape, generator=generator, device=device, dtype=dtype) def prepare_audio_latents( @@ -249,6 +219,7 @@ def prepare_audio_latents( device: torch.device | None = None, generator: torch.Generator | list[torch.Generator] | None = None, latents: torch.Tensor | None = None, + latents_normalized: bool = False, ) -> tuple[torch.Tensor, int, int]: original_latent_length = audio_latent_length latent_mel_bins = num_mel_bins // pipeline.audio_vae_mel_compression_ratio @@ -260,11 +231,12 @@ def prepare_audio_latents( latents = pack_audio_latents(latents) if latents.ndim != 3: raise ValueError(f"Provided `latents` has shape {latents.shape}, expected [batch, seq, features].") - latents = normalize_audio_latents( - latents, - pipeline.audio_vae.latents_mean, - pipeline.audio_vae.latents_std, - ) + if not latents_normalized: + latents = normalize_audio_latents( + latents, + pipeline.audio_vae.latents_mean, + pipeline.audio_vae.latents_std, + ) latents = create_noised_state(latents, noise_scale, generator) if latents.shape[1] not in {original_latent_length, padded_latent_length}: @@ -283,11 +255,11 @@ def prepare_audio_latents( latents = torch.cat([latents, padding], dim=1) return latents.to(device=device, dtype=dtype), original_latent_length, padded_latent_length - shape = (batch_size, num_channels_latents, padded_latent_length, latent_mel_bins) + shape = (batch_size, padded_latent_length, num_channels_latents * latent_mel_bins) if isinstance(generator, list) and len(generator) != batch_size: raise ValueError( f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" f" size of {batch_size}. Make sure the batch size matches the length of the generators." ) latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) - return pack_audio_latents(latents), original_latent_length, padded_latent_length + return latents, original_latent_length, padded_latent_length diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_recipes.py b/vllm_omni/diffusion/models/ltx2/ltx2_recipes.py index 6491b4a5f..a70f349ac 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_recipes.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_recipes.py @@ -1,20 +1,175 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""User-visible one-stage defaults for the LTX model family.""" +"""Declarative execution recipes for the LTX model family.""" from dataclasses import dataclass +from typing import Literal + +from .ltx2_guidance import LTXGuidanceSpec, LTXModalityGuidance + +LTX_DEFAULT_NEGATIVE_PROMPT = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) + +LTX_DISTILLED_SIGMAS = (1.0, 0.99375, 0.9875, 0.98125, 0.975, 0.909375, 0.725, 0.421875, 0.0) +LTX_STAGE_2_DISTILLED_SIGMAS = (0.909375, 0.725, 0.421875, 0.0) @dataclass(frozen=True) -class LTXOneStageRecipe: +class LTXPhaseRecipe: + """One denoise phase and the transition used to construct its input.""" + + name: str + guidance: LTXGuidanceSpec + spatial_downscale: int = 1 + sigmas: tuple[float, ...] | None = None + noise_scale: float = 0.0 + input_transform: Literal["initial", "spatial_upsample"] = "initial" + allow_guidance_override: bool = True + use_official_sigma_schedule: bool = True + + def __post_init__(self) -> None: + if self.spatial_downscale < 1: + raise ValueError("LTX phase spatial_downscale must be positive.") + if self.sigmas is not None and len(self.sigmas) < 2: + raise ValueError("An explicit LTX sigma schedule must contain at least two denoise sigmas.") + + @property + def num_inference_steps(self) -> int | None: + return None if self.sigmas is None else len(self.sigmas) - 1 + + +@dataclass(frozen=True) +class LTXPipelineRecipe: + """Request defaults, ordered phases, output routing, and request capabilities.""" + + phases: tuple[LTXPhaseRecipe, ...] height: int = 512 width: int = 768 num_frames: int = 121 frame_rate: float = 24.0 num_inference_steps: int = 40 - guidance_scale: float = 4.0 + negative_prompt: str = LTX_DEFAULT_NEGATIVE_PROMPT + video_output_phase: int = -1 + audio_output_phase: int = -1 + allow_request_sigmas: bool = True + allow_request_latents: bool = True + allow_negative_prompt: bool = True + fixed_num_inference_steps: bool = False + + def __post_init__(self) -> None: + if not self.phases: + raise ValueError("An LTX pipeline recipe must contain at least one phase.") + if self.phases[0].input_transform != "initial": + raise ValueError("The first LTX phase must use the initial request input.") + phase_count = len(self.phases) + for output_phase in (self.video_output_phase, self.audio_output_phase): + if not -phase_count <= output_phase < phase_count: + raise ValueError(f"LTX output phase {output_phase} is outside a {phase_count}-phase recipe.") + + @property + def request_guidance(self) -> LTXGuidanceSpec: + return self.phases[0].guidance + + @property + def max_spatial_downscale(self) -> int: + return max(phase.spatial_downscale for phase in self.phases) -LTX2_ONE_STAGE_RECIPE = LTXOneStageRecipe() -LTX23_ONE_STAGE_RECIPE = LTXOneStageRecipe() +def _official_guidance(stg_block: int) -> LTXGuidanceSpec: + return LTXGuidanceSpec( + video=LTXModalityGuidance( + cfg_scale=3.0, + stg_scale=1.0, + modality_scale=3.0, + rescale_scale=0.7, + stg_blocks=(stg_block,), + ), + audio=LTXModalityGuidance( + cfg_scale=7.0, + stg_scale=1.0, + modality_scale=3.0, + rescale_scale=0.7, + stg_blocks=(stg_block,), + ), + ) + + +LTX2_ONE_STAGE_RECIPE = LTXPipelineRecipe( + phases=(LTXPhaseRecipe(name="generate", guidance=_official_guidance(29)),), +) +LTX23_ONE_STAGE_RECIPE = LTXPipelineRecipe( + num_inference_steps=30, + phases=(LTXPhaseRecipe(name="generate", guidance=_official_guidance(28)),), +) +LTX_POSITIVE_ONLY_RECIPE = LTXPipelineRecipe( + phases=( + LTXPhaseRecipe( + name="generate", + guidance=LTXGuidanceSpec.positive_only(), + use_official_sigma_schedule=False, + ), + ), +) +LTX2_DISTILLED_TWO_STAGE_RECIPE = LTXPipelineRecipe( + # Official distilled arguments describe the final output. Stage 1 applies + # spatial_downscale=2 and therefore runs at 512x768. + height=1024, + width=1536, + num_inference_steps=len(LTX_DISTILLED_SIGMAS) - 1, + negative_prompt="", + phases=( + LTXPhaseRecipe( + name="generate_lowres", + guidance=LTXGuidanceSpec.positive_only(), + spatial_downscale=2, + sigmas=LTX_DISTILLED_SIGMAS, + noise_scale=1.0, + allow_guidance_override=False, + use_official_sigma_schedule=False, + ), + LTXPhaseRecipe( + name="refine", + guidance=LTXGuidanceSpec.positive_only(), + sigmas=LTX_STAGE_2_DISTILLED_SIGMAS, + noise_scale=LTX_STAGE_2_DISTILLED_SIGMAS[0], + input_transform="spatial_upsample", + allow_guidance_override=False, + use_official_sigma_schedule=False, + ), + ), + video_output_phase=1, + audio_output_phase=1, + allow_request_sigmas=False, + allow_request_latents=False, + allow_negative_prompt=False, + fixed_num_inference_steps=True, +) + + +_PIPELINE_RECIPES: dict[tuple[str, str], LTXPipelineRecipe] = { + ("one_stage", "2"): LTX2_ONE_STAGE_RECIPE, + ("one_stage", "2.3"): LTX23_ONE_STAGE_RECIPE, + ("distilled_two_stage", "2"): LTX2_DISTILLED_TWO_STAGE_RECIPE, + ("dmd2", "2"): LTX_POSITIVE_ONLY_RECIPE, + ("dmd2", "2.3"): LTX_POSITIVE_ONLY_RECIPE, +} + + +def resolve_ltx_pipeline_recipe(pipeline_kind: str, model_version: str) -> LTXPipelineRecipe: + """Resolve execution independently from component loading.""" + try: + return _PIPELINE_RECIPES[(pipeline_kind, model_version)] + except KeyError as exc: + raise ValueError(f"Unsupported LTX pipeline kind/version: {pipeline_kind!r}/{model_version!r}.") from exc diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_request.py b/vllm_omni/diffusion/models/ltx2/ltx2_request.py index d8a4356b3..e2eef19df 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_request.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_request.py @@ -1,17 +1,21 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Request normalization shared by LTX pipeline variants.""" +"""Request normalization and validation shared by LTX pipeline variants.""" from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import Any import torch +from vllm_omni.diffusion.data import DiffusionOutput from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch +from .ltx2_guidance import LTXGuidanceSpec, LTXModalityGuidance +from .ltx2_recipes import LTXPipelineRecipe + @dataclass class LTXRequestInputs: @@ -24,8 +28,7 @@ class LTXRequestInputs: num_frames: int frame_rate: float num_inference_steps: int - guidance_scale: float - guidance_rescale: float + guidance: LTXGuidanceSpec num_videos_per_prompt: int generator: torch.Generator | list[torch.Generator] | None latents: torch.Tensor | None @@ -38,6 +41,65 @@ class LTXRequestInputs: decode_noise_scale: float | list[float] | None output_type: str max_sequence_length: int + audio_latents_normalized: bool = False + + @property + def guidance_scale(self) -> float: + """Compatibility view for callers that only understand video CFG.""" + return self.guidance.video.cfg_scale + + +def validate_pipeline_request( + request_inputs: LTXRequestInputs, + *, + pipeline_recipe: LTXPipelineRecipe, + vae_spatial_compression_ratio: int, + vae_temporal_compression_ratio: int, + pipeline_name: str, + request_sigmas: list[float] | None = None, +) -> None: + """Validate the request capabilities declared by an execution recipe.""" + alignment = vae_spatial_compression_ratio * pipeline_recipe.max_spatial_downscale + if request_inputs.height % alignment != 0 or request_inputs.width % alignment != 0: + raise ValueError( + f"{pipeline_name} resolution must be divisible by {alignment}, got " + f"{request_inputs.width}x{request_inputs.height}." + ) + + if request_inputs.num_frames < 1 or (request_inputs.num_frames - 1) % vae_temporal_compression_ratio != 0: + raise ValueError( + f"{pipeline_name} `num_frames` must be {vae_temporal_compression_ratio} * k + 1, " + f"got {request_inputs.num_frames}." + ) + + if not pipeline_recipe.allow_request_latents and ( + request_inputs.latents is not None or request_inputs.audio_latents is not None + ): + raise ValueError(f"{pipeline_name} does not accept request-provided video or audio latents.") + + if not pipeline_recipe.allow_request_sigmas and request_sigmas is not None: + raise ValueError(f"{pipeline_name} uses fixed phase sigma schedules.") + + if pipeline_recipe.fixed_num_inference_steps and ( + request_inputs.num_inference_steps != pipeline_recipe.num_inference_steps + ): + raise ValueError( + f"{pipeline_name} uses {pipeline_recipe.num_inference_steps} fixed Stage 1 denoise steps; " + f"got {request_inputs.num_inference_steps}." + ) + + negative_prompt = request_inputs.negative_prompt + has_negative_prompt = ( + any(bool(value) for value in negative_prompt) if isinstance(negative_prompt, list) else bool(negative_prompt) + ) + if not pipeline_recipe.allow_negative_prompt and ( + has_negative_prompt or request_inputs.negative_prompt_embeds is not None + ): + raise ValueError(f"{pipeline_name} uses positive-only guidance and does not accept negative conditioning.") + + first_phase = pipeline_recipe.phases[0] + if not first_phase.allow_guidance_override and request_inputs.guidance != first_phase.guidance: + raise ValueError(f"{pipeline_name} uses fixed positive-only guidance and cannot override it.") def _unwrap_request_tensor(value: Any) -> Any: @@ -65,11 +127,187 @@ def _get_audio_latents_from_sampling(sampling: Any) -> torch.Tensor | None: return sampling.extra_args.get("audio_latents") +def _get_extra_arg(sampling: Any, *names: str) -> Any: + extra_args = sampling.extra_args or {} + for name in names: + value = extra_args.get(name) + if value is not None: + return value + return None + + +def _normalize_stg_blocks(value: Any, default: tuple[int, ...]) -> tuple[int, ...]: + if value is None: + return default + if isinstance(value, int): + return (value,) + if not isinstance(value, (list, tuple)): + raise ValueError(f"STG blocks must be an integer or a list of integers, got {value!r}.") + return tuple(int(item) for item in value) + + +def _resolve_negative_prompts( + prompts: list[Any], + fallback: str | list[str], +) -> str | list[str]: + if not prompts: + return fallback + if isinstance(fallback, list) and len(fallback) != len(prompts): + raise ValueError( + f"`negative_prompt` has batch size {len(fallback)}, but the request batch has size {len(prompts)}." + ) + + resolved = [] + for index, item in enumerate(prompts): + request_value = None if isinstance(item, str) else item.get("negative_prompt") + if request_value is not None: + resolved.append(request_value) + else: + resolved.append(fallback[index] if isinstance(fallback, list) else fallback) + return resolved + + +def _resolve_modality_guidance( + sampling: Any, + modality: str, + default: LTXModalityGuidance, +) -> LTXModalityGuidance: + cfg_aliases = (f"{modality}_cfg_scale", f"{modality}_cfg_guidance_scale") + stg_aliases = (f"{modality}_stg_scale", f"{modality}_stg_guidance_scale") + modality_aliases = ( + ("video_modality_scale", "a2v_guidance_scale") + if modality == "video" + else ("audio_modality_scale", "v2a_guidance_scale") + ) + cfg_scale = _get_extra_arg(sampling, *cfg_aliases) + stg_scale = _get_extra_arg(sampling, *stg_aliases) + modality_scale = _get_extra_arg(sampling, *modality_aliases) + rescale_scale = _get_extra_arg(sampling, f"{modality}_rescale_scale") + stg_blocks = _get_extra_arg(sampling, f"{modality}_stg_blocks") + return replace( + default, + cfg_scale=default.cfg_scale if cfg_scale is None else float(cfg_scale), + stg_scale=default.stg_scale if stg_scale is None else float(stg_scale), + modality_scale=default.modality_scale if modality_scale is None else float(modality_scale), + rescale_scale=default.rescale_scale if rescale_scale is None else float(rescale_scale), + stg_blocks=_normalize_stg_blocks(stg_blocks, default.stg_blocks), + ) + + +def _resolve_guidance_spec( + sampling: Any, + default: LTXGuidanceSpec, + *, + guidance_scale: float | None, +) -> LTXGuidanceSpec: + video = _resolve_modality_guidance(sampling, "video", default.video) + audio = _resolve_modality_guidance(sampling, "audio", default.audio) + + common_cfg = sampling.guidance_scale if sampling.guidance_scale_provided else guidance_scale + if common_cfg is not None: + video = replace(video, cfg_scale=float(common_cfg)) + audio = replace(audio, cfg_scale=float(common_cfg)) + + return LTXGuidanceSpec(video=video, audio=audio) + + class LTXRequestMixin: """Normalize serving requests without coupling them to a model version.""" supports_request_batch = False - supports_guidance_rescale = False + + @staticmethod + def _validate_forward_options( + *, + timesteps: list[int] | None, + guidance_rescale: float | None, + noise_scale: float, + return_dict: bool, + attention_kwargs: dict[str, Any] | None, + ) -> None: + if timesteps is not None: + raise ValueError("LTX does not support `timesteps`; use `sigmas` for a custom schedule.") + if guidance_rescale not in (None, 0.0): + raise ValueError( + "LTX does not support the common `guidance_rescale`; use `video_rescale_scale` and " + "`audio_rescale_scale`." + ) + if noise_scale != 0.0: + raise ValueError("LTX only supports `noise_scale=0.0`.") + if return_dict is not True: + raise ValueError("LTX only supports `return_dict=True`.") + if attention_kwargs is not None: + raise ValueError("LTX does not support public `attention_kwargs`.") + + @torch.no_grad() + def forward( + self, + req: DiffusionRequestBatch, + *, + image: Any | None = None, + prompt: str | list[str] | None = None, + negative_prompt: str | list[str] | None = None, + height: int | None = None, + width: int | None = None, + num_frames: int | None = None, + frame_rate: float | None = None, + num_inference_steps: int | None = None, + sigmas: list[float] | None = None, + timesteps: list[int] | None = None, + guidance_scale: float | None = None, + guidance_rescale: float | None = None, + noise_scale: float = 0.0, + num_videos_per_prompt: int | None = 1, + generator: torch.Generator | list[torch.Generator] | None = None, + latents: torch.Tensor | None = None, + audio_latents: torch.Tensor | None = None, + prompt_embeds: torch.Tensor | None = None, + negative_prompt_embeds: torch.Tensor | None = None, + prompt_attention_mask: torch.Tensor | None = None, + negative_prompt_attention_mask: torch.Tensor | None = None, + decode_timestep: float | list[float] = 0.0, + decode_noise_scale: float | list[float] | None = None, + output_type: str = "np", + return_dict: bool = True, + attention_kwargs: dict[str, Any] | None = None, + max_sequence_length: int | None = None, + ) -> DiffusionOutput | list[DiffusionOutput]: + """Validate and normalize the public LTX request surface.""" + self._validate_forward_options( + timesteps=timesteps, + guidance_rescale=guidance_rescale, + noise_scale=noise_scale, + return_dict=return_dict, + attention_kwargs=attention_kwargs, + ) + if image is not None and not self.support_image_input: + raise ValueError(f"{self.__class__.__name__} does not support `image` input.") + + return self._forward_request( + req, + image=image, + prompt=prompt, + negative_prompt=negative_prompt, + height=height, + width=width, + num_frames=num_frames, + frame_rate=frame_rate, + num_inference_steps=num_inference_steps, + sigmas=sigmas, + guidance_scale=guidance_scale, + num_videos_per_prompt=num_videos_per_prompt, + generator=generator, + latents=latents, + audio_latents=audio_latents, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + prompt_attention_mask=prompt_attention_mask, + negative_prompt_attention_mask=negative_prompt_attention_mask, + decode_timestep=decode_timestep, + decode_noise_scale=decode_noise_scale, + output_type=output_type, + max_sequence_length=max_sequence_length, + ) @staticmethod def _resolve_request_sigmas( @@ -128,8 +366,7 @@ class LTXRequestMixin: num_frames: int | None, frame_rate: float | None, num_inference_steps: int | None, - timesteps: list[int] | None, - guidance_scale: float, + guidance_scale: float | None, num_videos_per_prompt: int | None, generator: torch.Generator | list[torch.Generator] | None, latents: torch.Tensor | None, @@ -142,38 +379,55 @@ class LTXRequestMixin: decode_noise_scale: float | list[float] | None, output_type: str, max_sequence_length: int | None, - guidance_rescale: float = 0.0, ) -> LTXRequestInputs: sampling_params_list = req.sampling_params_list - sampling = sampling_params_list[0] - prompt = [item if isinstance(item, str) else (item.get("prompt") or "") for item in req.prompts] or prompt - if all(isinstance(item, str) or item.get("negative_prompt") is None for item in req.prompts): - negative_prompt = None - elif req.prompts: - negative_prompt = [ - "" if isinstance(item, str) else (item.get("negative_prompt") or "") for item in req.prompts - ] + for sampling in sampling_params_list: + if sampling.timesteps is not None: + raise ValueError("LTX does not support `timesteps`; use `sigmas` for a custom schedule.") + if sampling.guidance_rescale: + raise ValueError( + "LTX does not support the common `guidance_rescale`; use `video_rescale_scale` and " + "`audio_rescale_scale`." + ) + if _get_extra_arg(sampling, "flow_shift") is not None: + raise ValueError("LTX does not support `flow_shift`; use `sigmas` for a custom schedule.") - height = sampling.height or height or self.one_stage_recipe.height - width = sampling.width or width or self.one_stage_recipe.width - num_frames = sampling.num_frames or num_frames or self.one_stage_recipe.num_frames - frame_rate = sampling.resolved_frame_rate or frame_rate or self.one_stage_recipe.frame_rate - num_inference_steps = ( - sampling.num_inference_steps or num_inference_steps or self.one_stage_recipe.num_inference_steps + sampling = sampling_params_list[0] + is_dummy_run = req.is_dummy_run() + prompt = [item if isinstance(item, str) else (item.get("prompt") or "") for item in req.prompts] or prompt + negative_prompt = _resolve_negative_prompts( + req.prompts, + negative_prompt if negative_prompt is not None else self.pipeline_recipe.negative_prompt, ) - if timesteps is None: - num_inference_steps = max(int(num_inference_steps), 2) - elif len(timesteps) < 2: - raise ValueError("`timesteps` must contain at least 2 values for FlowMatchEulerDiscreteScheduler.") + + height = sampling.height or height or self.pipeline_recipe.height + width = sampling.width or width or self.pipeline_recipe.width + num_frames = sampling.num_frames or num_frames or self.pipeline_recipe.num_frames + frame_rate = sampling.resolved_frame_rate or frame_rate or self.pipeline_recipe.frame_rate + num_inference_steps = ( + sampling.num_inference_steps or num_inference_steps or self.pipeline_recipe.num_inference_steps + ) + if is_dummy_run and self.pipeline_recipe.fixed_num_inference_steps: + num_inference_steps = self.pipeline_recipe.num_inference_steps + num_inference_steps = max(int(num_inference_steps), 2) num_videos_per_prompt = ( sampling.num_outputs_per_prompt if sampling.num_outputs_per_prompt > 0 else num_videos_per_prompt or 1 ) max_sequence_length = sampling.max_sequence_length or max_sequence_length or self.tokenizer_max_length - if sampling.guidance_scale_provided: - guidance_scale = sampling.guidance_scale - if self.supports_guidance_rescale and sampling.guidance_rescale is not None: - guidance_rescale = sampling.guidance_rescale + guidance_specs = [ + _resolve_guidance_spec( + item, + self.pipeline_recipe.request_guidance, + guidance_scale=guidance_scale, + ) + for item in sampling_params_list + ] + guidance = guidance_specs[0] + if is_dummy_run and not self.pipeline_recipe.phases[0].allow_guidance_override: + guidance = self.pipeline_recipe.request_guidance + if any(item != guidance for item in guidance_specs[1:]): + raise ValueError("Batched LTX requests must use identical video/audio guidance parameters.") if self.supports_request_batch: if generator is None: @@ -235,6 +489,10 @@ class LTXRequestMixin: ] if any(value is not None for value in request_negative_masks): negative_prompt_attention_mask = torch.stack(request_negative_masks) # type: ignore[arg-type] + if prompt_embeds is not None: + prompt = None + if negative_prompt_embeds is not None: + negative_prompt = None if sampling.decode_timestep is not None: decode_timestep = sampling.decode_timestep @@ -251,8 +509,7 @@ class LTXRequestMixin: num_frames=int(num_frames), frame_rate=float(frame_rate), num_inference_steps=int(num_inference_steps), - guidance_scale=guidance_scale, - guidance_rescale=guidance_rescale, + guidance=guidance, num_videos_per_prompt=int(num_videos_per_prompt), generator=generator, latents=latents, diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_pipeline_runtime.py b/vllm_omni/diffusion/models/ltx2/ltx2_runtime.py similarity index 59% rename from vllm_omni/diffusion/models/ltx2/ltx2_pipeline_runtime.py rename to vllm_omni/diffusion/models/ltx2/ltx2_runtime.py index 08bf20330..b5bb18a9b 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_pipeline_runtime.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_runtime.py @@ -1,12 +1,13 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Shared runtime surface for LTX pipeline variants.""" +"""Shared recipe-driven runtime for LTX pipeline variants.""" from __future__ import annotations from collections.abc import Iterable from contextlib import nullcontext +from dataclasses import replace from typing import Any, ClassVar import torch @@ -21,22 +22,41 @@ from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch, split_diffusion_output_by_request +from vllm_omni.platforms import current_omni_platform from . import ltx2_latents as latent_ops -from .ltx2_components import LTXComponentProfile, initialize_pipeline_components +from .ltx2_components import ( + LTXComponentProfile, + detect_ltx_model_version, + initialize_pipeline_components, + resolve_ltx_component_profile, +) from .ltx2_conditioning import LTXPromptContext, LTXTextConditioningMixin from .ltx2_denoise import ( LTXDenoiseContext, LTXForwardContext, LTXPhaseExecutor, LTXPhaseResult, - VideoAudioScheduler, build_transformer_kwargs, step_denoised_latents, ) -from .ltx2_guidance import LTXGuidanceStrategy -from .ltx2_recipes import LTXOneStageRecipe -from .ltx2_request import LTXRequestInputs, LTXRequestMixin +from .ltx2_guidance import ( + LTX_GUIDANCE_EXECUTOR, + LTXGuidanceExecutor, + LTXGuidancePlan, + LTXGuidanceSpec, + LTXModalityGuidance, +) +from .ltx2_recipes import ( + LTXPhaseRecipe, + LTXPipelineRecipe, + resolve_ltx_pipeline_recipe, +) +from .ltx2_request import ( + LTXRequestInputs, + LTXRequestMixin, + validate_pipeline_request, +) def _expand_per_prompt_decode_value( @@ -92,7 +112,7 @@ def _prepare_decode_timestep_conditioning( ) -class LTXPipelineRuntime( +class LTXRuntime( LTXRequestMixin, LTXTextConditioningMixin, nn.Module, @@ -101,27 +121,146 @@ class LTXPipelineRuntime( SupportsComponentDiscovery, DiffusionPipelineProfilerMixin, ): - """Shared Omni runtime for explicitly composed LTX denoise phases.""" + """Shared Omni runtime for recipe-driven LTX denoise phases.""" + pipeline_kind: ClassVar[str] = "one_stage" component_profile: ClassVar[LTXComponentProfile] - guidance_strategy: ClassVar[LTXGuidanceStrategy] - one_stage_recipe: ClassVar[LTXOneStageRecipe] + pipeline_recipe: ClassVar[LTXPipelineRecipe] + guidance_executor: ClassVar[LTXGuidanceExecutor] = LTX_GUIDANCE_EXECUTOR supports_request_batch = False connector_batches_cfg = False distributed_video_decode = True support_image_input = False - dummy_run_num_frames = 2 + dummy_run_num_frames = 1 preserve_sp_padded_audio_duration = False reports_stage_durations = False def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = "") -> None: del prefix + self.model_version = detect_ltx_model_version(od_config.model) + self.component_profile = resolve_ltx_component_profile(self.pipeline_kind, self.model_version) + self.pipeline_recipe = resolve_ltx_pipeline_recipe(self.pipeline_kind, self.model_version) + self._dit_modules = list(self.component_profile.dit_modules) + self._encoder_modules = list(self.component_profile.encoder_modules) + self._vae_modules = list(self.component_profile.vae_modules) + self._resident_modules = list(self.component_profile.resident_modules) + if self.model_version == "2.3": + self.preserve_sp_padded_audio_duration = True + self.reports_stage_durations = True super().__init__() + self._guidance_plan = LTXGuidancePlan.build(self.pipeline_recipe.request_guidance) initialize_pipeline_components(self, od_config) self.setup_diffusion_pipeline_profiler( enable_diffusion_pipeline_profiler=self.od_config.enable_diffusion_pipeline_profiler ) + def _forward_request( + self, + req: DiffusionRequestBatch, + *, + image: Any | None = None, + prompt: str | list[str] | None = None, + negative_prompt: str | list[str] | None = None, + height: int | None = None, + width: int | None = None, + num_frames: int | None = None, + frame_rate: float | None = None, + num_inference_steps: int | None = None, + sigmas: list[float] | None = None, + guidance_scale: float | None = None, + num_videos_per_prompt: int | None = 1, + generator: torch.Generator | list[torch.Generator] | None = None, + latents: torch.Tensor | None = None, + audio_latents: torch.Tensor | None = None, + prompt_embeds: torch.Tensor | None = None, + negative_prompt_embeds: torch.Tensor | None = None, + prompt_attention_mask: torch.Tensor | None = None, + negative_prompt_attention_mask: torch.Tensor | None = None, + decode_timestep: float | list[float] = 0.0, + decode_noise_scale: float | list[float] | None = None, + output_type: str = "np", + max_sequence_length: int | None = None, + ) -> DiffusionOutput | list[DiffusionOutput]: + request_inputs = self._resolve_request_inputs( + req, + prompt=prompt, + negative_prompt=negative_prompt, + height=height, + width=width, + num_frames=num_frames, + frame_rate=frame_rate, + num_inference_steps=num_inference_steps, + guidance_scale=guidance_scale, + num_videos_per_prompt=num_videos_per_prompt, + generator=generator, + latents=latents, + audio_latents=audio_latents, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + prompt_attention_mask=prompt_attention_mask, + negative_prompt_attention_mask=negative_prompt_attention_mask, + decode_timestep=decode_timestep, + decode_noise_scale=decode_noise_scale, + output_type=output_type, + max_sequence_length=max_sequence_length, + ) + image = self._resolve_request_image(req, image, request_inputs) + request_sigmas = self._resolve_request_sigmas(req, sigmas) + validate_pipeline_request( + request_inputs, + pipeline_recipe=self.pipeline_recipe, + vae_spatial_compression_ratio=self.vae_spatial_compression_ratio, + vae_temporal_compression_ratio=self.vae_temporal_compression_ratio, + pipeline_name=self.__class__.__name__, + request_sigmas=request_sigmas, + ) + return self._run_recipe(req, request_inputs, request_sigmas=request_sigmas, image=image) + + def _run_recipe( + self, + req: DiffusionRequestBatch, + request_inputs: LTXRequestInputs, + *, + request_sigmas: list[float] | None, + image: Any | None = None, + ) -> DiffusionOutput | list[DiffusionOutput]: + """Execute one- and multi-phase recipes through the same control flow.""" + phase_results: list[LTXPhaseResult] = [] + prompt_context = None + + for phase_recipe in self.pipeline_recipe.phases: + self._enter_phase(phase_recipe) + phase_inputs = self._build_phase_inputs( + request_inputs, + phase_recipe, + phase_results[-1] if phase_results else None, + ) + phase_result = self.run_phase( + req, + phase_inputs, + noise_scale=phase_recipe.noise_scale, + sigmas=list(phase_recipe.sigmas) if phase_recipe.sigmas is not None else request_sigmas, + timesteps=None, + attention_kwargs=None, + phase_recipe=phase_recipe, + image=image, + prompt_context=prompt_context, + ) + phase_results.append(phase_result) + prompt_context = phase_result.forward_context.prompt_context + + final_context = phase_results[-1].forward_context + output_phase = LTXPhaseResult( + forward_context=final_context, + video=phase_results[self.pipeline_recipe.video_output_phase].video, + audio=phase_results[self.pipeline_recipe.audio_output_phase].audio, + ) + return self.decode_phase(output_phase) + + def _enter_phase(self, phase: LTXPhaseRecipe) -> None: + """Hook for a future phase-weight strategy.""" + del phase + def prepare_latents( self, batch_size: int = 1, @@ -160,6 +299,7 @@ class LTXPipelineRuntime( device: torch.device | None = None, generator: torch.Generator | list[torch.Generator] | None = None, latents: torch.Tensor | None = None, + latents_normalized: bool = False, ) -> tuple[torch.Tensor, int, int]: return latent_ops.prepare_audio_latents( self, @@ -172,19 +312,16 @@ class LTXPipelineRuntime( device, generator, latents, + latents_normalized, ) @property def guidance_scale(self): - return self._guidance_scale - - @property - def guidance_rescale(self): - return self._guidance_rescale + return self._guidance_plan.spec.video.cfg_scale @property def do_classifier_free_guidance(self): - return self._guidance_scale is not None and self._guidance_scale > 1.0 + return self._guidance_plan.spec.do_cfg @property def interrupt(self): @@ -212,21 +349,29 @@ class LTXPipelineRuntime( ): if kwargs is not None: context = {**kwargs, **context} - return self.guidance_strategy.combine_cfg_noise( - self, - positive_noise_pred, - negative_noise_pred, - true_cfg_scale, - cfg_normalize, - context, + del cfg_normalize + required = ("video_latents", "audio_latents", "video_sigma", "audio_sigma") + if any(context.get(name) is None for name in required): + raise ValueError("LTX x0-space CFG requires video/audio latents and sigmas.") + video_pos, audio_pos = positive_noise_pred + video_neg, audio_neg = negative_noise_pred + return ( + self.guidance_executor.combine_cfg_velocity( + context["video_latents"], + video_pos, + video_neg, + context["video_sigma"], + true_cfg_scale, + ), + self.guidance_executor.combine_cfg_velocity( + context["audio_latents"], + audio_pos, + audio_neg, + context["audio_sigma"], + true_cfg_scale, + ), ) - def predict_noise_with_parallel_cfg(self, *args, **kwargs): - predict_parallel_cfg = getattr(self.guidance_strategy, "predict_parallel_cfg", None) - if predict_parallel_cfg is None: - raise NotImplementedError("The selected LTX guidance strategy does not implement parallel CFG.") - return predict_parallel_cfg(self, *args, **kwargs) - def _synchronize_cfg_parallel_step_output( self, latents: tuple[torch.Tensor, torch.Tensor], @@ -238,23 +383,35 @@ class LTXPipelineRuntime( # CUDA async execution otherwise permits numerical drift to accumulate # across CFG-parallel denoise steps. latents = tuple(tensor.contiguous() for tensor in latents) - device = next((tensor.device for tensor in latents if tensor.is_cuda), None) - if device is not None: - torch.cuda.current_stream(device).synchronize() + current_omni_platform.synchronize() return latents def _setup_forward_runtime( self, + req: DiffusionRequestBatch, request_inputs: LTXRequestInputs, attention_kwargs: dict[str, Any] | None, ) -> bool: - self._guidance_scale = request_inputs.guidance_scale - self._guidance_rescale = request_inputs.guidance_rescale + self._guidance_plan = LTXGuidancePlan.build(request_inputs.guidance) del attention_kwargs self._interrupt = False cfg_world_size = get_classifier_free_guidance_world_size() + if ( + req.is_dummy_run() + and cfg_world_size == 2 + and self.do_classifier_free_guidance + and not self._guidance_plan.cfg_parallel_compatible + ): + # The single-device warmup follows the full recipe. CFG2 executes + # the same per-modality CFG scales without STG/modality passes. + self._guidance_plan = LTXGuidancePlan.build( + LTXGuidanceSpec( + video=LTXModalityGuidance(cfg_scale=self._guidance_plan.spec.video.cfg_scale), + audio=LTXModalityGuidance(cfg_scale=self._guidance_plan.spec.audio.cfg_scale), + ) + ) if self.do_classifier_free_guidance: - self.guidance_strategy.validate_cfg_world_size(cfg_world_size) + self.guidance_executor.validate_cfg_world_size(self._guidance_plan, cfg_world_size) return self.do_classifier_free_guidance and cfg_world_size > 1 def _check_forward_inputs( @@ -325,11 +482,8 @@ class LTXPipelineRuntime( vae_decode_needs_all_ranks = False is_distributed_vae_enabled = getattr(self.vae, "is_distributed_enabled", None) if self.distributed_video_decode and dist_initialized and callable(is_distributed_vae_enabled): - try: - # Distributed tiled decode is collective, so every rank must enter it. - vae_decode_needs_all_ranks = bool(is_distributed_vae_enabled()) - except Exception: - pass + # Distributed tiled decode is collective, so every rank must enter it. + vae_decode_needs_all_ranks = bool(is_distributed_vae_enabled()) should_decode_video = not self.distributed_video_decode or is_output_rank or vae_decode_needs_all_ranks if should_decode_video: @@ -361,16 +515,16 @@ class LTXPipelineRuntime( image: Any | None = None, ) -> tuple[torch.Tensor, torch.Tensor | None]: latents = self.prepare_latents( - prompt_context.batch_size * request_inputs.num_videos_per_prompt, - self.transformer.config.in_channels, - request_inputs.height, - request_inputs.width, - request_inputs.num_frames, - noise_scale, - torch.float32, - device, - request_inputs.generator, - request_inputs.latents, + batch_size=prompt_context.batch_size * request_inputs.num_videos_per_prompt, + num_channels_latents=self.transformer.config.in_channels, + height=request_inputs.height, + width=request_inputs.width, + num_frames=request_inputs.num_frames, + noise_scale=noise_scale, + dtype=prompt_context.positive_connector_prompt_embeds.dtype, + device=device, + generator=request_inputs.generator, + latents=request_inputs.latents, ) return latents, None @@ -416,10 +570,11 @@ class LTXPipelineRuntime( audio_latent_length=audio_num_frames, num_mel_bins=num_mel_bins, noise_scale=noise_scale, - dtype=torch.float32, + dtype=prompt_context.positive_connector_audio_prompt_embeds.dtype, device=device, generator=request_inputs.generator, latents=request_inputs.audio_latents, + latents_normalized=request_inputs.audio_latents_normalized, ) return audio_latents, original_num_frames, padded_num_frames, latent_mel_bins @@ -439,29 +594,37 @@ class LTXPipelineRuntime( padded_length = latent_ops.get_sp_padded_audio_latent_length(requested_length, int(sp_size)) return requested_length if provided_length in {requested_length, padded_length} else provided_length - def _make_video_audio_scheduler( - self, - audio_scheduler: Any, - latent_num_frames: int, - latent_height: int, - latent_width: int, - ) -> Any: - return VideoAudioScheduler(self.scheduler, audio_scheduler) - def _prepare_denoise_context_for_cfg( self, forward_ctx: LTXForwardContext, denoise_ctx: LTXDenoiseContext, ) -> LTXDenoiseContext: - return self.guidance_strategy.prepare_denoise_context(self, forward_ctx, denoise_ctx) + return self.guidance_executor.prepare_denoise_context( + self._guidance_plan, + forward_ctx.cfg_parallel_ready, + denoise_ctx, + ) def _denoise_timestep_kwargs( self, ts: torch.Tensor, forward_ctx: LTXForwardContext, denoise_ctx: LTXDenoiseContext, + *, + video_token_count: int, + audio_token_count: int, ) -> dict[str, torch.Tensor]: - return self.guidance_strategy.timestep_kwargs(ts, forward_ctx, denoise_ctx) + del forward_ctx, denoise_ctx + return self.guidance_executor.timestep_kwargs(ts, video_token_count, audio_token_count) + + def _video_guidance_model_sigma( + self, + sigma: torch.Tensor, + denoise_ctx: LTXDenoiseContext, + ) -> torch.Tensor: + """Return the video timestep used to convert velocity predictions to x0.""" + del denoise_ctx + return sigma def _build_transformer_kwargs( self, @@ -475,6 +638,7 @@ class LTXPipelineRuntime( encoder_attention_mask: torch.Tensor | None, audio_encoder_attention_mask: torch.Tensor | None, ts: torch.Tensor, + attention_kwargs: dict[str, Any] | None = None, ) -> dict[str, Any]: return build_transformer_kwargs( self, @@ -487,6 +651,7 @@ class LTXPipelineRuntime( encoder_attention_mask=encoder_attention_mask, audio_encoder_attention_mask=audio_encoder_attention_mask, ts=ts, + attention_kwargs=attention_kwargs, ) def _predict_noise_for_step( @@ -497,8 +662,9 @@ class LTXPipelineRuntime( forward_ctx: LTXForwardContext, denoise_ctx: LTXDenoiseContext, ) -> tuple[torch.Tensor, torch.Tensor]: - return self.guidance_strategy.predict_noise( + return self.guidance_executor.predict_noise( self, + self._guidance_plan, index, timestep, state, @@ -562,7 +728,6 @@ class LTXPipelineRuntime( ) audio_latents = latent_ops.unpack_audio_latents( audio_latents, - forward_ctx.original_audio_num_frames, num_mel_bins=forward_ctx.latent_mel_bins, ) return latents, audio_latents @@ -576,6 +741,7 @@ class LTXPipelineRuntime( sigmas: list[float] | None, timesteps: list[int] | None, attention_kwargs: dict[str, Any] | None, + phase_recipe: LTXPhaseRecipe, image: Any | None = None, prompt_context: LTXPromptContext | None = None, ) -> LTXPhaseResult: @@ -588,10 +754,84 @@ class LTXPipelineRuntime( sigmas=sigmas, timesteps=timesteps, attention_kwargs=attention_kwargs, + phase_recipe=phase_recipe, image=image, prompt_context=prompt_context, ) + def _build_phase_inputs( + self, + request_inputs: LTXRequestInputs, + phase: LTXPhaseRecipe, + previous_phase: LTXPhaseResult | None, + ) -> LTXRequestInputs: + """Resolve one phase from immutable request inputs and prior AV state.""" + divisor = phase.spatial_downscale + if request_inputs.height % divisor != 0 or request_inputs.width % divisor != 0: + raise ValueError( + f"LTX phase {phase.name!r} cannot scale resolution " + f"{request_inputs.width}x{request_inputs.height} by {divisor}." + ) + height = request_inputs.height // divisor + width = request_inputs.width // divisor + latents = request_inputs.latents + audio_latents = request_inputs.audio_latents + audio_latents_normalized = request_inputs.audio_latents_normalized + decode_timestep = request_inputs.decode_timestep + decode_noise_scale = request_inputs.decode_noise_scale + + if phase.input_transform == "spatial_upsample": + if previous_phase is None: + raise ValueError(f"LTX phase {phase.name!r} requires a previous phase to upsample.") + if previous_phase.video.ndim != 5: + raise ValueError(f"LTX spatial upsampling expects a 5D video latent, got {previous_phase.video.shape}.") + latents = self._spatial_upsample_phase(previous_phase.video) + if previous_phase.audio_for_next_phase is not None: + audio_latents = previous_phase.audio_for_next_phase + audio_latents_normalized = True + else: + audio_latents = previous_phase.audio + audio_latents_normalized = False + decode_timestep = 0.0 + decode_noise_scale = None + + expected_shape = latent_ops.resolve_video_latent_shape( + height, + width, + request_inputs.num_frames, + vae_spatial_compression_ratio=self.vae_spatial_compression_ratio, + vae_temporal_compression_ratio=self.vae_temporal_compression_ratio, + ) + if latents.shape[2:] != expected_shape: + raise ValueError( + f"LTX phase {phase.name!r} upsampler produced {tuple(latents.shape[2:])}, expected " + f"{expected_shape} for {width}x{height}." + ) + elif phase.input_transform != "initial": + raise ValueError(f"Unsupported LTX phase input transform: {phase.input_transform!r}.") + + guidance = request_inputs.guidance if phase.allow_guidance_override else phase.guidance + return replace( + request_inputs, + height=height, + width=width, + num_inference_steps=phase.num_inference_steps or request_inputs.num_inference_steps, + guidance=guidance, + latents=latents, + audio_latents=audio_latents, + audio_latents_normalized=audio_latents_normalized, + decode_timestep=decode_timestep, + decode_noise_scale=decode_noise_scale, + ) + + def _spatial_upsample_phase(self, latents: torch.Tensor) -> torch.Tensor: + """Apply the recipe-managed spatial upsampler to unpacked latents.""" + latent_upsampler = getattr(self, "latent_upsampler", None) + if latent_upsampler is None: + raise RuntimeError("This LTX pipeline recipe requires a spatial latent upsampler.") + dtype = getattr(latent_upsampler, "dtype", latents.dtype) + return latent_upsampler(latents.to(device=self.device, dtype=dtype)) + def decode_phase(self, phase: LTXPhaseResult) -> DiffusionOutput | list[DiffusionOutput]: """Decode one completed phase and restore per-request outputs.""" forward_ctx = phase.forward_context diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_transformer.py b/vllm_omni/diffusion/models/ltx2/ltx2_transformer.py index 3d1a2d663..95724bd19 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_transformer.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_transformer.py @@ -57,6 +57,9 @@ _RMSNORM_INIT_PARAMS = inspect.signature(RMSNorm.__init__).parameters def _make_rms_norm(hidden_size: int, *, eps: float, elementwise_affine: bool) -> nn.Module: """Bridge diffusers' RMSNorm API onto vLLM's `has_weight` variant.""" + if not elementwise_affine: + return nn.RMSNorm(hidden_size, eps=eps, elementwise_affine=False) + kwargs: dict[str, Any] = {"eps": eps} if "elementwise_affine" in _RMSNORM_INIT_PARAMS: kwargs["elementwise_affine"] = elementwise_affine @@ -79,10 +82,14 @@ def _make_rms_norm(hidden_size: int, *, eps: float, elementwise_affine: bool) -> def apply_interleaved_rotary_emb(x: torch.Tensor, freqs: tuple[torch.Tensor, torch.Tensor]) -> torch.Tensor: cos, sin = freqs + if cos.dtype != x.dtype: + cos = cos.to(dtype=x.dtype) + if sin.dtype != x.dtype: + sin = sin.to(dtype=x.dtype) # Concrete pair count instead of -1 keeps SDPA shape static under torch.compile. x_real, x_imag = x.unflatten(2, (x.shape[2] // 2, 2)).unbind(-1) # [B, S, C // 2] x_rotated = torch.stack([-x_imag, x_real], dim=-1).flatten(2) - out = (x.float() * cos + x_rotated.float() * sin).to(x.dtype) + out = x * cos + x_rotated * sin return out @@ -94,6 +101,10 @@ def apply_split_rotary_emb( ) -> torch.Tensor: # `head_dim` is plumbed in (not inferred via `-1`) so SDPA shape stays static under torch.compile. cos, sin = freqs + if cos.dtype != x.dtype: + cos = cos.to(dtype=x.dtype) + if sin.dtype != x.dtype: + sin = sin.to(dtype=x.dtype) x_dtype = x.dtype needs_reshape = False @@ -112,7 +123,7 @@ def apply_split_rotary_emb( r = last // 2 # (..., 2, r) - split_x = x.reshape(*x.shape[:-1], 2, r).float() # Explicitly upcast to float + split_x = x.reshape(*x.shape[:-1], 2, r) first_x = split_x[..., :1, :] # (..., 1, r) second_x = split_x[..., 1:, :] # (..., 1, r) @@ -327,6 +338,23 @@ class TensorParallelRMSNorm(nn.Module): return out.to(dtype=x_dtype) +def to_ltx_padding_mask(attention_mask: torch.Tensor) -> torch.Tensor: + """Convert an additive/expanded LTX mask to a 2D padding mask.""" + if attention_mask.ndim > 2: + if attention_mask.is_floating_point(): + valid = attention_mask >= 0 + else: + valid = attention_mask.to(torch.bool) + batch_size = valid.shape[0] + key_length = valid.shape[-1] + attention_mask = valid.reshape(batch_size, -1, key_length).all(dim=1) + if attention_mask.is_floating_point(): + attention_mask = attention_mask >= 0 + if attention_mask.dtype != torch.bool: + attention_mask = attention_mask.to(torch.bool) + return attention_mask + + class LTX2AudioVideoAttnProcessor: r""" Processor for implementing attention (SDPA is used by default if you're using PyTorch 2.0) for the LTX-2.0 model. @@ -344,24 +372,6 @@ class LTX2AudioVideoAttnProcessor: "Please upgrade your PyTorch installation." ) - @staticmethod - def _to_padding_mask(attention_mask: torch.Tensor) -> torch.Tensor: - # Convert additive/expanded masks into a 2D padding mask for flash-attn. - if attention_mask.ndim > 2: - if attention_mask.is_floating_point(): - valid = attention_mask >= 0 - else: - valid = attention_mask.to(torch.bool) - b = valid.shape[0] - key_len = valid.shape[-1] - valid = valid.reshape(b, -1, key_len).all(dim=1) - attention_mask = valid - if attention_mask.is_floating_point(): - attention_mask = attention_mask >= 0 - if attention_mask.dtype != torch.bool: - attention_mask = attention_mask.to(torch.bool) - return attention_mask - @staticmethod def _is_sp_enabled() -> bool: if not is_forward_context_available(): @@ -390,12 +400,12 @@ class LTX2AudioVideoAttnProcessor: # For cross-attention, encoder sequence length != query length, so drop the mask. if encoder_hidden_states is not None and encoder_hidden_states.shape[1] != hidden_states.shape[1]: return None - return self._to_padding_mask(attention_mask) + return to_ltx_padding_mask(attention_mask) attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1]) if attn.attn.attn_backend.get_name().upper() == "FLASH_ATTN": - attention_mask = self._to_padding_mask(attention_mask) + attention_mask = to_ltx_padding_mask(attention_mask) return attention_mask @staticmethod @@ -467,6 +477,7 @@ class LTX2AudioVideoAttnProcessor: attention_mask: torch.Tensor | None = None, query_rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None, key_rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None, + perturbation_mask: torch.Tensor | None = None, ) -> torch.Tensor: is_self_attention = encoder_hidden_states is None batch_size, sequence_length, _ = hidden_states.shape if is_self_attention else encoder_hidden_states.shape @@ -509,17 +520,21 @@ class LTX2AudioVideoAttnProcessor: elif attn.rope_type == "split": query = apply_split_rotary_emb(query, query_rotary_emb, head_dim=attn.head_dim) key = apply_split_rotary_emb( - key, key_rotary_emb if key_rotary_emb is not None else query_rotary_emb, head_dim=attn.head_dim + key, + key_rotary_emb if key_rotary_emb is not None else query_rotary_emb, + head_dim=attn.head_dim, ) query = query.unflatten(2, (attn.heads, attn.head_dim)) key = key.unflatten(2, (attn.heads, attn.head_dim)) - value = value.unflatten(2, (attn.heads, attn.head_dim)) + value_heads = value.unflatten(2, (attn.heads, attn.head_dim)) attn_metadata = AttentionMetadata(attn_mask=attention_mask) if attention_mask is not None else None - hidden_states = attn.attn(query, key, value, attn_metadata) + hidden_states = attn.attn(query, key, value_heads, attn_metadata) hidden_states = hidden_states.flatten(2, 3) hidden_states = hidden_states.to(query.dtype) + if perturbation_mask is not None: + hidden_states = hidden_states * perturbation_mask + value * (1 - perturbation_mask) # LTX-2.3: per-head gated attention if attn.to_gate_logits is not None: @@ -560,6 +575,7 @@ class LTX2Attention(torch.nn.Module): norm_elementwise_affine: bool = True, rope_type: str = "interleaved", apply_gated_attention: bool = False, + pack_qkv: bool = True, processor=None, quant_config: "QuantizationConfig | None" = None, prefix: str = "", @@ -591,7 +607,7 @@ class LTX2Attention(torch.nn.Module): self.to_q = None self.to_k = None self.to_v = None - if cross_attention_dim is None: + if cross_attention_dim is None and pack_qkv: self.to_qkv = QKVParallelLinear( hidden_size=query_dim, head_size=self.head_dim, @@ -847,6 +863,9 @@ class LTX2VideoTransformerBlock(nn.Module): qk_norm=qk_norm, rope_type=rope_type, apply_gated_attention=audio_gated_attn, + # The official checkpoint evaluates audio Q/K/V separately. Fusing + # the bf16 GEMMs changes denoise numerics enough to accumulate. + pack_qkv=False, quant_config=quant_config, prefix=f"{prefix}.audio_attn1" if prefix else "audio_attn1", ) @@ -992,8 +1011,10 @@ class LTX2VideoTransformerBlock(nn.Module): v2a_cross_attention_mask: torch.Tensor | None = None, use_a2v_cross_attention: bool = True, use_v2a_cross_attention: bool = True, - perturbation_mask: torch.Tensor | None = None, - all_perturbed: bool | None = None, + video_self_attention_perturbation_mask: torch.Tensor | None = None, + audio_self_attention_perturbation_mask: torch.Tensor | None = None, + a2v_cross_attention_perturbation_mask: torch.Tensor | None = None, + v2a_cross_attention_perturbation_mask: torch.Tensor | None = None, ) -> torch.Tensor: batch_size = hidden_states.size(0) @@ -1012,6 +1033,7 @@ class LTX2VideoTransformerBlock(nn.Module): encoder_hidden_states=None, query_rotary_emb=video_rotary_emb, attention_mask=self_attention_mask, + perturbation_mask=video_self_attention_perturbation_mask, ) hidden_states = hidden_states + attn_hidden_states * gate_msa @@ -1031,6 +1053,7 @@ class LTX2VideoTransformerBlock(nn.Module): encoder_hidden_states=None, query_rotary_emb=audio_rotary_emb, attention_mask=audio_self_attention_mask, + perturbation_mask=audio_self_attention_perturbation_mask, ) audio_hidden_states = audio_hidden_states + attn_audio_hidden_states * audio_gate_msa @@ -1123,6 +1146,8 @@ class LTX2VideoTransformerBlock(nn.Module): key_rotary_emb=ca_audio_rotary_emb, attention_mask=a2v_cross_attention_mask, ) + if a2v_cross_attention_perturbation_mask is not None: + a2v_attn_hidden_states = a2v_attn_hidden_states * a2v_cross_attention_perturbation_mask hidden_states = hidden_states + a2v_gate * a2v_attn_hidden_states # 3.3. Video-to-Audio Cross Attention: Q: Audio; K,V: Video @@ -1141,6 +1166,8 @@ class LTX2VideoTransformerBlock(nn.Module): key_rotary_emb=ca_video_rotary_emb, attention_mask=v2a_cross_attention_mask, ) + if v2a_cross_attention_perturbation_mask is not None: + v2a_attn_hidden_states = v2a_attn_hidden_states * v2a_cross_attention_perturbation_mask audio_hidden_states = audio_hidden_states + v2a_gate * v2a_attn_hidden_states # 4. Feedforward @@ -1345,7 +1372,10 @@ class LTX2AudioVideoRotaryPosEmbed(nn.Module): return self.prepare_audio_coords(*args, **kwargs) def forward( - self, coords: torch.Tensor, device: str | torch.device | None = None + self, + coords: torch.Tensor, + device: str | torch.device | None = None, + out_dtype: torch.dtype | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: device = device or coords.device @@ -1421,6 +1451,9 @@ class LTX2AudioVideoRotaryPosEmbed(nn.Module): cos_freqs = torch.swapaxes(cos_freq, 1, 2) # (B,H,T,D//2) sin_freqs = torch.swapaxes(sin_freq, 1, 2) # (B,H,T,D//2) + if out_dtype is not None: + cos_freqs = cos_freqs.to(out_dtype) + sin_freqs = sin_freqs.to(out_dtype) return cos_freqs, sin_freqs @@ -1488,8 +1521,11 @@ class LTX2VideoTransformer3DModel(nn.Module): # Shard prompt embeds across sequence "encoder_hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3, split_output=False), "audio_encoder_hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3, split_output=False), - # Shard video timestep when provided as (B, seq_len) + # Keep per-token video/audio timestep embeddings aligned with + # their corresponding latent sequence shards. Official LTX + # multi-guidance supplies both tensors as (B, seq_len). "timestep": SequenceParallelInput(split_dim=1, expected_dims=2, split_output=False), + "audio_timestep": SequenceParallelInput(split_dim=1, expected_dims=2, split_output=False), }, "rope": { 0: SequenceParallelInput(split_dim=rope_split_dim, expected_dims=rope_expected_dims, split_output=True), @@ -1856,6 +1892,12 @@ class LTX2VideoTransformer3DModel(nn.Module): # Determine timestep for audio. audio_timestep = audio_timestep if audio_timestep is not None else timestep + def batch_step(step: torch.Tensor) -> torch.Tensor: + return step[:, 0] if step.ndim > 1 else step + + sigma = sigma if sigma is not None else batch_step(timestep) + audio_sigma = audio_sigma if audio_sigma is not None else batch_step(audio_timestep) + # convert encoder_attention_mask to a bias the same way we do for attention_mask if encoder_attention_mask is not None and encoder_attention_mask.ndim == 2: encoder_attention_mask = (1 - encoder_attention_mask.to(hidden_states.dtype)) * -10000.0 @@ -1877,12 +1919,22 @@ class LTX2VideoTransformer3DModel(nn.Module): batch_size, audio_num_frames, audio_hidden_states.device ) - video_rotary_emb = self.rope(video_coords, device=hidden_states.device) - audio_rotary_emb = self.audio_rope(audio_coords, device=audio_hidden_states.device) + video_rotary_emb = self.rope(video_coords, device=hidden_states.device, out_dtype=hidden_states.dtype) + audio_rotary_emb = self.audio_rope( + audio_coords, + device=audio_hidden_states.device, + out_dtype=audio_hidden_states.dtype, + ) - video_cross_attn_rotary_emb = self.cross_attn_rope(video_coords[:, 0:1, :], device=hidden_states.device) + video_cross_attn_rotary_emb = self.cross_attn_rope( + video_coords[:, 0:1, :], + device=hidden_states.device, + out_dtype=hidden_states.dtype, + ) audio_cross_attn_rotary_emb = self.cross_attn_audio_rope( - audio_coords[:, 0:1, :], device=audio_hidden_states.device + audio_coords[:, 0:1, :], + device=audio_hidden_states.device, + out_dtype=audio_hidden_states.dtype, ) # 2. Patchify input projections @@ -1920,7 +1972,7 @@ class LTX2VideoTransformer3DModel(nn.Module): hidden_dtype=hidden_states.dtype, ) video_cross_attn_a2v_gate, _ = self.av_cross_attn_video_a2v_gate( - timestep.flatten() * timestep_cross_attn_gate_scale_factor, + audio_sigma.flatten() * timestep_cross_attn_gate_scale_factor, batch_size=batch_size, hidden_dtype=hidden_states.dtype, ) @@ -1935,7 +1987,7 @@ class LTX2VideoTransformer3DModel(nn.Module): hidden_dtype=audio_hidden_states.dtype, ) audio_cross_attn_v2a_gate, _ = self.av_cross_attn_audio_v2a_gate( - audio_timestep.flatten() * timestep_cross_attn_gate_scale_factor, + sigma.flatten() * timestep_cross_attn_gate_scale_factor, batch_size=batch_size, hidden_dtype=audio_hidden_states.dtype, ) @@ -1945,7 +1997,6 @@ class LTX2VideoTransformer3DModel(nn.Module): audio_cross_attn_v2a_gate = audio_cross_attn_v2a_gate.view(batch_size, -1, audio_cross_attn_v2a_gate.shape[-1]) # 3.3. LTX-2.3: Compute prompt modulation from sigma - audio_sigma = audio_sigma if audio_sigma is not None else sigma if self.prompt_modulation and sigma is not None: temb_prompt, _ = self.prompt_adaln(sigma.flatten(), batch_size=batch_size, hidden_dtype=hidden_states.dtype) temb_prompt_audio, _ = self.audio_prompt_adaln( @@ -1967,8 +2018,17 @@ class LTX2VideoTransformer3DModel(nn.Module): audio_encoder_hidden_states = self.audio_caption_projection(audio_encoder_hidden_states) audio_encoder_hidden_states = audio_encoder_hidden_states.view(batch_size, -1, audio_hidden_states.size(-1)) + perturbation_kwargs = (attention_kwargs or {}).get("ltx_perturbation_kwargs", {}) + + def perturbation_mask_for(name: str, block_idx: int) -> torch.Tensor | None: + mask = perturbation_kwargs.get(f"{name}_mask") + blocks = perturbation_kwargs.get(f"{name}_blocks") + if mask is None or (blocks is not None and block_idx not in blocks): + return None + return mask + # 5. Run transformer blocks - for block in self.transformer_blocks: + for block_idx, block in enumerate(self.transformer_blocks): block_kwargs = { "temb": temb, "temb_audio": temb_audio, @@ -1984,6 +2044,10 @@ class LTX2VideoTransformer3DModel(nn.Module): "ca_audio_rotary_emb": audio_cross_attn_rotary_emb, "encoder_attention_mask": encoder_attention_mask, "audio_encoder_attention_mask": audio_encoder_attention_mask, + "video_self_attention_perturbation_mask": perturbation_mask_for("video_self_attention", block_idx), + "audio_self_attention_perturbation_mask": perturbation_mask_for("audio_self_attention", block_idx), + "a2v_cross_attention_perturbation_mask": perturbation_mask_for("a2v_cross_attention", block_idx), + "v2a_cross_attention_perturbation_mask": perturbation_mask_for("v2a_cross_attention", block_idx), } # NOTE: We intentionally pass the first 4 args here positionally, # because CacheDiT expects the first 2 args to be hidden_states @@ -2066,7 +2130,10 @@ class LTX2VideoTransformer3DModel(nn.Module): for param_name, weight_name, shard_id in stacked_params_mapping: if weight_name not in name: continue - name = name.replace(weight_name, param_name) + packed_name = name.replace(weight_name, param_name) + if packed_name not in params_dict: + continue + name = packed_name param = params_dict[name] weight_loader = param.weight_loader weight_loader(param, loaded_weight, shard_id) diff --git a/vllm_omni/diffusion/models/ltx2/pipeline_ltx2.py b/vllm_omni/diffusion/models/ltx2/pipeline_ltx2.py index 5de1f4862..ff66af5e1 100644 --- a/vllm_omni/diffusion/models/ltx2/pipeline_ltx2.py +++ b/vllm_omni/diffusion/models/ltx2/pipeline_ltx2.py @@ -5,250 +5,56 @@ from __future__ import annotations -from typing import Any, ClassVar +from typing import ClassVar -import torch - -from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig +from vllm_omni.diffusion.data import OmniDiffusionConfig from vllm_omni.diffusion.models.dmd2 import DMD2PipelineMixin -from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch -from .ltx2_components import ( - LTX2_COMPONENT_PROFILE, - LTX23_COMPONENT_PROFILE, - LTXComponentProfile, -) +from .ltx2_components import LTX2_COMPONENT_PROFILE from .ltx2_components import ( get_ltx2_post_process_func as get_ltx2_post_process_func, # noqa: F401 ) from .ltx2_conditioning import LTXI2VConditioningMixin -from .ltx2_guidance import ( - LTX_LEGACY_VELOCITY_GUIDANCE, - LTX_OFFICIAL_X0_GUIDANCE, +from .ltx2_recipes import ( + LTX2_ONE_STAGE_RECIPE, + LTX_POSITIVE_ONLY_RECIPE, ) -from .ltx2_pipeline_runtime import LTXPipelineRuntime -from .ltx2_recipes import LTX2_ONE_STAGE_RECIPE, LTX23_ONE_STAGE_RECIPE, LTXOneStageRecipe -from .ltx2_request import LTXRequestInputs +from .ltx2_runtime import LTXRuntime -class LTXOneStagePipeline(LTXPipelineRuntime): - """Single execution path configured by model-version and task entries.""" +class LTX2Pipeline(LTXI2VConditioningMixin, LTXRuntime): + """LTX-2 family one-stage entry, configured from checkpoint metadata.""" - component_profile: ClassVar[LTXComponentProfile] - one_stage_recipe: ClassVar[LTXOneStageRecipe] - - supports_request_batch = True - supports_guidance_rescale = False - - def _forward_impl( - self, - req: DiffusionRequestBatch, - request_inputs: LTXRequestInputs, - *, - noise_scale: float, - sigmas: list[float] | None, - timesteps: list[int] | None, - attention_kwargs: dict[str, Any] | None, - image: Any | None = None, - ) -> DiffusionOutput | list[DiffusionOutput]: - phase = self.run_phase( - req, - request_inputs, - noise_scale=noise_scale, - sigmas=sigmas, - timesteps=timesteps, - attention_kwargs=attention_kwargs, - image=image, - ) - return self.decode_phase(phase) - - @torch.no_grad() - def forward( - self, - req: DiffusionRequestBatch, - prompt: str | list[str] | None = None, - negative_prompt: str | list[str] | None = None, - height: int | None = None, - width: int | None = None, - num_frames: int | None = None, - frame_rate: float | None = None, - num_inference_steps: int | None = None, - sigmas: list[float] | None = None, - timesteps: list[int] | None = None, - guidance_scale: float = 4.0, - guidance_rescale: float = 0.0, - noise_scale: float = 0.0, - num_videos_per_prompt: int | None = 1, - generator: torch.Generator | list[torch.Generator] | None = None, - latents: torch.Tensor | None = None, - audio_latents: torch.Tensor | None = None, - prompt_embeds: torch.Tensor | None = None, - negative_prompt_embeds: torch.Tensor | None = None, - prompt_attention_mask: torch.Tensor | None = None, - negative_prompt_attention_mask: torch.Tensor | None = None, - decode_timestep: float | list[float] = 0.0, - decode_noise_scale: float | list[float] | None = None, - output_type: str = "np", - return_dict: bool = True, - attention_kwargs: dict[str, Any] | None = None, - max_sequence_length: int | None = None, - *, - image: Any | None = None, - ) -> DiffusionOutput | list[DiffusionOutput]: - del return_dict - sigmas = self._resolve_request_sigmas(req, sigmas) - request_inputs = self._resolve_request_inputs( - req, - prompt=prompt, - negative_prompt=negative_prompt, - height=height, - width=width, - num_frames=num_frames, - frame_rate=frame_rate, - num_inference_steps=num_inference_steps, - timesteps=timesteps, - guidance_scale=self.one_stage_recipe.guidance_scale if guidance_scale is None else guidance_scale, - guidance_rescale=guidance_rescale, - num_videos_per_prompt=num_videos_per_prompt, - generator=generator, - latents=latents, - audio_latents=audio_latents, - prompt_embeds=prompt_embeds, - negative_prompt_embeds=negative_prompt_embeds, - prompt_attention_mask=prompt_attention_mask, - negative_prompt_attention_mask=negative_prompt_attention_mask, - decode_timestep=decode_timestep, - decode_noise_scale=decode_noise_scale, - output_type=output_type, - max_sequence_length=max_sequence_length, - ) - image = self._resolve_request_image(req, image, request_inputs) - forward_kwargs = { - "noise_scale": noise_scale, - "sigmas": sigmas, - "timesteps": timesteps, - "attention_kwargs": attention_kwargs, - } - if self.support_image_input: - forward_kwargs["image"] = image - return self._forward_impl( - req, - request_inputs, - **forward_kwargs, - ) - - -class LTX2Pipeline(LTXOneStagePipeline): - """LTX2 one-stage text-to-video entry.""" - - supports_guidance_rescale = True + pipeline_kind = "one_stage" + unified_text_image_entry = True component_profile = LTX2_COMPONENT_PROFILE - guidance_strategy = LTX_LEGACY_VELOCITY_GUIDANCE - one_stage_recipe = LTX2_ONE_STAGE_RECIPE + pipeline_recipe = LTX2_ONE_STAGE_RECIPE _dit_modules: ClassVar[list[str]] = list(component_profile.dit_modules) _encoder_modules: ClassVar[list[str]] = list(component_profile.encoder_modules) _vae_modules: ClassVar[list[str]] = list(component_profile.vae_modules) _resident_modules: ClassVar[list[str]] = list(component_profile.resident_modules) - - -class LTX23Pipeline(LTXOneStagePipeline): - """LTX2.3 one-stage text-to-video entry.""" - - connector_batches_cfg = True - preserve_sp_padded_audio_duration = True - reports_stage_durations = True - component_profile = LTX23_COMPONENT_PROFILE - guidance_strategy = LTX_OFFICIAL_X0_GUIDANCE - one_stage_recipe = LTX23_ONE_STAGE_RECIPE - _dit_modules: ClassVar[list[str]] = list(component_profile.dit_modules) - _encoder_modules: ClassVar[list[str]] = list(component_profile.encoder_modules) - _vae_modules: ClassVar[list[str]] = list(component_profile.vae_modules) - _resident_modules: ClassVar[list[str]] = list(component_profile.resident_modules) - - @torch.no_grad() - def forward( - self, - req: DiffusionRequestBatch, - prompt: str | list[str] | None = None, - negative_prompt: str | list[str] | None = None, - height: int | None = None, - width: int | None = None, - num_frames: int | None = None, - frame_rate: float | None = None, - num_inference_steps: int | None = None, - sigmas: list[float] | None = None, - timesteps: list[int] | None = None, - guidance_scale: float = 4.0, - noise_scale: float = 0.0, - num_videos_per_prompt: int | None = 1, - generator: torch.Generator | list[torch.Generator] | None = None, - latents: torch.Tensor | None = None, - audio_latents: torch.Tensor | None = None, - prompt_embeds: torch.Tensor | None = None, - negative_prompt_embeds: torch.Tensor | None = None, - prompt_attention_mask: torch.Tensor | None = None, - negative_prompt_attention_mask: torch.Tensor | None = None, - decode_timestep: float | list[float] = 0.0, - decode_noise_scale: float | list[float] | None = None, - output_type: str = "np", - return_dict: bool = True, - attention_kwargs: dict[str, Any] | None = None, - max_sequence_length: int | None = None, - *, - image: Any | None = None, - ) -> DiffusionOutput | list[DiffusionOutput]: - """Preserve the pre-refactor LTX2.3 positional argument order.""" - return super().forward( - req, - prompt=prompt, - negative_prompt=negative_prompt, - height=height, - width=width, - num_frames=num_frames, - frame_rate=frame_rate, - num_inference_steps=num_inference_steps, - sigmas=sigmas, - timesteps=timesteps, - guidance_scale=guidance_scale, - guidance_rescale=0.0, - noise_scale=noise_scale, - num_videos_per_prompt=num_videos_per_prompt, - generator=generator, - latents=latents, - audio_latents=audio_latents, - prompt_embeds=prompt_embeds, - negative_prompt_embeds=negative_prompt_embeds, - prompt_attention_mask=prompt_attention_mask, - negative_prompt_attention_mask=negative_prompt_attention_mask, - decode_timestep=decode_timestep, - decode_noise_scale=decode_noise_scale, - output_type=output_type, - return_dict=return_dict, - attention_kwargs=attention_kwargs, - max_sequence_length=max_sequence_length, - image=image, - ) - - -class LTX2ImageToVideoPipeline(LTXI2VConditioningMixin, LTX2Pipeline): - """LTX2 one-stage image-to-video entry.""" - - -class LTX23ImageToVideoPipeline(LTXI2VConditioningMixin, LTX23Pipeline): - """LTX2.3 one-stage image-to-video entry.""" + supports_request_batch = True class LTX2T2VDMD2Pipeline(DMD2PipelineMixin, LTX2Pipeline): """LTX2 T2V entry for FastGen DMD2-distilled models.""" + pipeline_kind = "dmd2" + support_image_input = False + pipeline_recipe = LTX_POSITIVE_ONLY_RECIPE + def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""): super().__init__(od_config=od_config, prefix=prefix) self.__init_dmd2__() -class LTX2I2VDMD2Pipeline(DMD2PipelineMixin, LTX2ImageToVideoPipeline): +class LTX2I2VDMD2Pipeline(DMD2PipelineMixin, LTX2Pipeline): """LTX2 I2V entry for FastGen DMD2-distilled models.""" + pipeline_kind = "dmd2" + unified_text_image_entry = False + pipeline_recipe = LTX_POSITIVE_ONLY_RECIPE + def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""): super().__init__(od_config=od_config, prefix=prefix) self.__init_dmd2__() diff --git a/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_latent_upsample.py b/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_latent_upsample.py deleted file mode 100644 index 0c72a41d5..000000000 --- a/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_latent_upsample.py +++ /dev/null @@ -1,262 +0,0 @@ -import os - -import torch -import torch.nn as nn -from diffusers import AutoencoderKLLTX2Video -from diffusers.image_processor import PipelineImageInput -from diffusers.pipelines.ltx2.latent_upsampler import LTX2LatentUpsamplerModel -from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img import retrieve_latents -from diffusers.utils.torch_utils import randn_tensor -from diffusers.video_processor import VideoProcessor -from vllm.logger import init_logger - -from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig -from vllm_omni.diffusion.distributed.utils import get_local_device - -logger = init_logger(__name__) - - -class LTX2LatentUpsamplePipeline(nn.Module): - def __init__( - self, - od_config: OmniDiffusionConfig, - vae: AutoencoderKLLTX2Video, - latent_upsampler: LTX2LatentUpsamplerModel = None, - ) -> None: - super().__init__() - - if vae is None: - raise ValueError("vae must be provided") - self.vae = vae - - self.device = get_local_device() - model = od_config.model - local_files_only = os.path.exists(model) - - if latent_upsampler is None: - # Use cpu context to create latent upsampler. The code k[:, None] @ k[None, :] in - # diffuser's BlurDownsample is not supported on GPU as k is type of torch.Int64 - with torch.device("cpu"): - latent_upsampler = LTX2LatentUpsamplerModel.from_pretrained( - model, - subfolder="latent_upsampler", - torch_dtype=torch.bfloat16, - local_files_only=local_files_only, - ).to(self.device) - self.latent_upsampler = latent_upsampler - - self.vae_spatial_compression_ratio = ( - self.vae.spatial_compression_ratio if getattr(self, "vae", None) is not None else 32 - ) - self.vae_temporal_compression_ratio = ( - self.vae.temporal_compression_ratio if getattr(self, "vae", None) is not None else 8 - ) - self.video_processor = VideoProcessor(vae_scale_factor=self.vae_spatial_compression_ratio) - - def prepare_latents( - self, - video: torch.Tensor | None = None, - batch_size: int = 1, - num_frames: int = 121, - height: int = 512, - width: int = 768, - spatial_patch_size: int = 1, - temporal_patch_size: int = 1, - dtype: torch.dtype | None = None, - device: torch.device | None = None, - generator: torch.Generator | None = None, - latents: torch.Tensor | None = None, - ) -> torch.Tensor: - if latents is not None: - if latents.ndim == 3: - # Convert token seq [B, S, D] to latent video [B, C, F, H, W] - latent_num_frames = (num_frames - 1) // self.vae_temporal_compression_ratio + 1 - latent_height = height // self.vae_spatial_compression_ratio - latent_width = width // self.vae_spatial_compression_ratio - latents = self._unpack_latents( - latents, latent_num_frames, latent_height, latent_width, spatial_patch_size, temporal_patch_size - ) - return latents.to(device=device, dtype=dtype) - - video = video.to(device=device, dtype=self.vae.dtype) - if isinstance(generator, list): - if len(generator) != batch_size: - raise ValueError( - f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" - f" size of {batch_size}. Make sure the batch size matches the length of the generators." - ) - - init_latents = [ - retrieve_latents(self.vae.encode(video[i].unsqueeze(0)), generator[i]) for i in range(batch_size) - ] - else: - init_latents = [retrieve_latents(self.vae.encode(vid.unsqueeze(0)), generator) for vid in video] - - init_latents = torch.cat(init_latents, dim=0).to(dtype) - # NOTE: latent upsampler operates on the unnormalized latents, so don't normalize here - # init_latents = self._normalize_latents(init_latents, self.vae.latents_mean, self.vae.latents_std) - return init_latents - - def adain_filter_latent(self, latents: torch.Tensor, reference_latents: torch.Tensor, factor: float = 1.0): - result = latents.clone() - - for i in range(latents.size(0)): - for c in range(latents.size(1)): - r_sd, r_mean = torch.std_mean(reference_latents[i, c], dim=None) # index by original dim order - i_sd, i_mean = torch.std_mean(result[i, c], dim=None) - - result[i, c] = ((result[i, c] - i_mean) / i_sd) * r_sd + r_mean - - result = torch.lerp(latents, result, factor) - return result - - def tone_map_latents(self, latents: torch.Tensor, compression: float) -> torch.Tensor: - # Remap [0-1] to [0-0.75] and apply sigmoid compression in one shot - scale_factor = compression * 0.75 - abs_latents = torch.abs(latents) - - # Sigmoid compression: sigmoid shifts large values toward 0.2, small values stay ~1.0 - # When scale_factor=0, sigmoid term vanishes, when scale_factor=0.75, full effect - sigmoid_term = torch.sigmoid(4.0 * scale_factor * (abs_latents - 1.0)) - scales = 1.0 - 0.8 * scale_factor * sigmoid_term - - filtered = latents * scales - return filtered - - @staticmethod - # Copied from diffusers.pipelines.ltx2.pipeline_ltx2.LTX2Pipeline._denormalize_latents - def _denormalize_latents( - latents: torch.Tensor, latents_mean: torch.Tensor, latents_std: torch.Tensor, scaling_factor: float = 1.0 - ) -> torch.Tensor: - # Denormalize latents across the channel dimension [B, C, F, H, W] - latents_mean = latents_mean.view(1, -1, 1, 1, 1).to(latents.device, latents.dtype) - latents_std = latents_std.view(1, -1, 1, 1, 1).to(latents.device, latents.dtype) - latents = latents * latents_std / scaling_factor + latents_mean - return latents - - @staticmethod - # Copied from diffusers.pipelines.ltx2.pipeline_ltx2.LTX2Pipeline._unpack_latents - def _unpack_latents( - latents: torch.Tensor, num_frames: int, height: int, width: int, patch_size: int = 1, patch_size_t: int = 1 - ) -> torch.Tensor: - # Packed latents of shape [B, S, D] (S is the effective video sequence length, D is the effective feature dimensions) # noqa - # are unpacked and reshaped into a video tensor of shape [B, C, F, H, W]. This is the inverse operation of - # what happens in the `_pack_latents` method. - batch_size = latents.size(0) - latents = latents.reshape(batch_size, num_frames, height, width, -1, patch_size_t, patch_size, patch_size) - latents = latents.permute(0, 4, 1, 5, 2, 6, 3, 7).flatten(6, 7).flatten(4, 5).flatten(2, 3) - return latents - - def check_inputs(self, video, height, width, latents, tone_map_compression_ratio): - if height % self.vae_spatial_compression_ratio != 0 or width % self.vae_spatial_compression_ratio != 0: - raise ValueError(f"`height` and `width` have to be divisible by 32 but are {height} and {width}.") - - if video is not None and latents is not None: - raise ValueError("Only one of `video` or `latents` can be provided.") - if video is None and latents is None: - raise ValueError("One of `video` or `latents` has to be provided.") - - if not (0 <= tone_map_compression_ratio <= 1): - raise ValueError("`tone_map_compression_ratio` must be in the range [0, 1]") - - def forward( - self, - video: list[PipelineImageInput] | None = None, - height: int = 512, - width: int = 768, - num_frames: int = 121, - spatial_patch_size: int = 1, - temporal_patch_size: int = 1, - latents: torch.Tensor | None = None, - latents_normalized: bool = False, - decode_timestep: float | list[float] = 0.0, - decode_noise_scale: float | list[float] | None = None, - adain_factor: float = 0.0, - tone_map_compression_ratio: float = 0.0, - generator: torch.Generator | list[torch.Generator] | None = None, - output_type: str | None = "pil", - return_dict: bool = True, - ): - self.check_inputs( - video=video, - height=height, - width=width, - latents=latents, - tone_map_compression_ratio=tone_map_compression_ratio, - ) - - if video is not None: - # Batched video input is not yet tested/supported. TODO: take a look later - batch_size = 1 - else: - batch_size = latents.shape[0] - device = self.device - - if video is not None: - num_frames = len(video) - if num_frames % self.vae_temporal_compression_ratio != 1: - num_frames = num_frames // self.vae_temporal_compression_ratio * self.vae_temporal_compression_ratio + 1 - video = video[:num_frames] - logger.warning( - f"Video length expected to be of the form `k * {self.vae_temporal_compression_ratio} + 1` but is {len(video)}. Truncating to {num_frames} frames." # noqa - ) - video = self.video_processor.preprocess_video(video, height=height, width=width) - video = video.to(device=device, dtype=torch.float32) - - latents_supplied = latents is not None - latents = self.prepare_latents( - video=video, - batch_size=batch_size, - num_frames=num_frames, - height=height, - width=width, - spatial_patch_size=spatial_patch_size, - temporal_patch_size=temporal_patch_size, - dtype=torch.float32, - device=device, - generator=generator, - latents=latents, - ) - - if latents_supplied and latents_normalized: - latents = self._denormalize_latents( - latents, self.vae.latents_mean, self.vae.latents_std, self.vae.config.scaling_factor - ) - latents = latents.to(self.latent_upsampler.dtype) - latents_upsampled = self.latent_upsampler(latents) - - if adain_factor > 0.0: - latents = self.adain_filter_latent(latents_upsampled, latents, adain_factor) - else: - latents = latents_upsampled - - if tone_map_compression_ratio > 0.0: - latents = self.tone_map_latents(latents, tone_map_compression_ratio) - - if output_type == "latent": - video = latents - else: - if not self.vae.config.timestep_conditioning: - timestep = None - else: - noise = randn_tensor(latents.shape, generator=generator, device=device, dtype=latents.dtype) - if not isinstance(decode_timestep, list): - decode_timestep = [decode_timestep] * batch_size - if decode_noise_scale is None: - decode_noise_scale = decode_timestep - elif not isinstance(decode_noise_scale, list): - decode_noise_scale = [decode_noise_scale] * batch_size - - timestep = torch.tensor(decode_timestep, device=device, dtype=latents.dtype) - decode_noise_scale = torch.tensor(decode_noise_scale, device=device, dtype=latents.dtype)[ - :, None, None, None, None - ] - latents = (1 - decode_noise_scale) * latents + decode_noise_scale * noise - - video = self.vae.decode(latents, timestep, return_dict=False)[0] - video = self.video_processor.postprocess_video(video, output_type=output_type) - - if not return_dict: - return (video,) - - return DiffusionOutput(output=(video,)) diff --git a/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_two_stage.py b/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_two_stage.py index 8b786b817..0e5bbc558 100644 --- a/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_two_stage.py +++ b/vllm_omni/diffusion/models/ltx2/pipeline_ltx2_two_stage.py @@ -1,207 +1,27 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Two-stage entry points for the LTX model family.""" +"""Distilled two-stage entry points for the LTX model family.""" -from __future__ import annotations +from vllm_omni.diffusion.data import OmniDiffusionConfig -import os -from dataclasses import replace -from typing import Any, ClassVar - -import torch -from diffusers.pipelines.ltx2.utils import DISTILLED_SIGMA_VALUES, STAGE_2_DISTILLED_SIGMA_VALUES - -from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig -from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch - -from .ltx2_components import ( - LTX2_COMPONENT_PROFILE, -) +from .ltx2_components import LTX2_DISTILLED_COMPONENT_PROFILE from .ltx2_components import ( get_ltx2_post_process_func as get_ltx2_post_process_func, # noqa: F401 ) -from .ltx2_conditioning import LTXI2VConditioningMixin -from .ltx2_guidance import LTX_LEGACY_VELOCITY_GUIDANCE -from .ltx2_pipeline_runtime import LTXPipelineRuntime -from .ltx2_recipes import LTX2_ONE_STAGE_RECIPE -from .ltx2_request import LTXRequestInputs -from .pipeline_ltx2_latent_upsample import LTX2LatentUpsamplePipeline +from .ltx2_recipes import LTX2_DISTILLED_TWO_STAGE_RECIPE +from .pipeline_ltx2 import LTX2Pipeline -class LTX2TwoStagesPipeline(LTXPipelineRuntime): - """Legacy distilled-only LTX2 two-stage compatibility entry.""" +class LTX2DistilledPipeline(LTX2Pipeline): + """Unified LTX-2 distilled two-stage T2V/I2V entry.""" - component_profile = LTX2_COMPONENT_PROFILE - guidance_strategy = LTX_LEGACY_VELOCITY_GUIDANCE - one_stage_recipe = LTX2_ONE_STAGE_RECIPE + pipeline_kind = "distilled_two_stage" + component_profile = LTX2_DISTILLED_COMPONENT_PROFILE + pipeline_recipe = LTX2_DISTILLED_TWO_STAGE_RECIPE supports_request_batch = False - support_image_input = False - - _dit_modules: ClassVar[list[str]] = list(component_profile.dit_modules) - _encoder_modules: ClassVar[list[str]] = list(component_profile.encoder_modules) - _vae_modules: ClassVar[list[str]] = list(component_profile.vae_modules) - _resident_modules: ClassVar[list[str]] = list(component_profile.resident_modules) + support_image_input = True + unified_text_image_entry = True def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""): - model_path = od_config.model - self.distilled = "distilled" in os.path.basename(os.path.normpath(model_path)) - if not self.distilled: - raise NotImplementedError(f"{model_path} is not supported for {self.__class__.__name__}.") - super().__init__(od_config=od_config, prefix=prefix) - self.upsample_pipe = LTX2LatentUpsamplePipeline(vae=self.vae, od_config=od_config) - - def _run_two_stage( - self, - req: DiffusionRequestBatch, - request_inputs: LTXRequestInputs, - *, - noise_scale: float, - timesteps: list[int] | None, - attention_kwargs: dict[str, Any] | None, - image: Any | None = None, - ) -> DiffusionOutput | list[DiffusionOutput]: - stage1 = self.run_phase( - req, - request_inputs, - noise_scale=noise_scale, - sigmas=DISTILLED_SIGMA_VALUES if self.distilled else None, - timesteps=timesteps, - attention_kwargs=attention_kwargs, - image=image, - ) - upscaled_video_latent = self.upsample_pipe( - latents=stage1.video, - output_type="latent", - return_dict=False, - )[0] - - stage2_inputs = replace( - request_inputs, - num_inference_steps=3, - guidance_scale=1.0, - latents=upscaled_video_latent, - audio_latents=stage1.audio, - decode_timestep=0.0, - decode_noise_scale=None, - output_type="np", - ) - stage2 = self.run_phase( - req, - stage2_inputs, - noise_scale=STAGE_2_DISTILLED_SIGMA_VALUES[0], - sigmas=STAGE_2_DISTILLED_SIGMA_VALUES, - timesteps=None, - attention_kwargs=attention_kwargs, - prompt_context=stage1.forward_context.prompt_context, - ) - return self.decode_phase(stage2) - - @torch.no_grad() - def forward( - self, - req: DiffusionRequestBatch, - prompt: str | list[str] | None = None, - negative_prompt: str | list[str] | None = None, - height: int | None = None, - width: int | None = None, - num_frames: int | None = None, - frame_rate: float | None = None, - num_inference_steps: int | None = None, - timesteps: list[int] | None = None, - guidance_scale: float = 4.0, - guidance_rescale: float = 0.0, - noise_scale: float = 0.0, - num_videos_per_prompt: int | None = 1, - generator: torch.Generator | list[torch.Generator] | None = None, - latents: torch.Tensor | None = None, - audio_latents: torch.Tensor | None = None, - prompt_embeds: torch.Tensor | None = None, - negative_prompt_embeds: torch.Tensor | None = None, - prompt_attention_mask: torch.Tensor | None = None, - negative_prompt_attention_mask: torch.Tensor | None = None, - decode_timestep: float | list[float] = 0.0, - decode_noise_scale: float | list[float] | None = None, - output_type: str = "np", - return_dict: bool = True, - attention_kwargs: dict[str, Any] | None = None, - max_sequence_length: int | None = None, - *, - image: Any | None = None, - ) -> DiffusionOutput | list[DiffusionOutput]: - del return_dict - request_inputs = self._resolve_request_inputs( - req, - prompt=prompt, - negative_prompt=negative_prompt, - height=height, - width=width, - num_frames=num_frames, - frame_rate=frame_rate, - num_inference_steps=num_inference_steps, - timesteps=timesteps, - guidance_scale=( - getattr(getattr(self, "one_stage_recipe", None), "guidance_scale", 4.0) - if guidance_scale is None - else guidance_scale - ), - guidance_rescale=guidance_rescale, - num_videos_per_prompt=num_videos_per_prompt, - generator=generator, - latents=latents, - audio_latents=audio_latents, - prompt_embeds=prompt_embeds, - negative_prompt_embeds=negative_prompt_embeds, - prompt_attention_mask=prompt_attention_mask, - negative_prompt_attention_mask=negative_prompt_attention_mask, - decode_timestep=decode_timestep, - decode_noise_scale=decode_noise_scale, - output_type=output_type, - max_sequence_length=max_sequence_length, - ) - if self.support_image_input: - image = self._resolve_request_image(req, image, request_inputs) - return self._run_two_stage( - req, - request_inputs, - noise_scale=noise_scale, - timesteps=timesteps, - attention_kwargs=attention_kwargs, - image=image, - ) - - -class LTX2ImageToVideoTwoStagesPipeline(LTXI2VConditioningMixin, LTX2TwoStagesPipeline): - """LTX2 two-stage image-to-video entry.""" - - def forward( - self, - req: DiffusionRequestBatch, - image: Any | None = None, - prompt: str | list[str] | None = None, - negative_prompt: str | list[str] | None = None, - height: int | None = None, - width: int | None = None, - num_frames: int | None = None, - frame_rate: float | None = None, - num_inference_steps: int | None = None, - sigmas: list[float] | None = None, - *args: Any, - **kwargs: Any, - ) -> DiffusionOutput | list[DiffusionOutput]: - """Preserve the legacy I2V sigma slot, which was ignored by this pipeline.""" - del sigmas - return super().forward( - req, - image, - prompt, - negative_prompt, - height, - width, - num_frames, - frame_rate, - num_inference_steps, - *args, - **kwargs, - ) diff --git a/vllm_omni/diffusion/registry.py b/vllm_omni/diffusion/registry.py index a0b1e9a7b..aaedc7b00 100644 --- a/vllm_omni/diffusion/registry.py +++ b/vllm_omni/diffusion/registry.py @@ -71,20 +71,10 @@ _DIFFUSION_MODELS = { "pipeline_ltx2", "LTX2Pipeline", ), - "LTX2ImageToVideoPipeline": ( - "ltx2", - "pipeline_ltx2", - "LTX2ImageToVideoPipeline", - ), - "LTX2TwoStagesPipeline": ( + "LTX2DistilledPipeline": ( "ltx2", "pipeline_ltx2_two_stage", - "LTX2TwoStagesPipeline", - ), - "LTX2ImageToVideoTwoStagesPipeline": ( - "ltx2", - "pipeline_ltx2_two_stage", - "LTX2ImageToVideoTwoStagesPipeline", + "LTX2DistilledPipeline", ), "LTX2T2VDMD2Pipeline": ( "ltx2", @@ -96,16 +86,6 @@ _DIFFUSION_MODELS = { "pipeline_ltx2", "LTX2I2VDMD2Pipeline", ), - "LTX23Pipeline": ( - "ltx2", - "pipeline_ltx2", - "LTX23Pipeline", - ), - "LTX23ImageToVideoPipeline": ( - "ltx2", - "pipeline_ltx2", - "LTX23ImageToVideoPipeline", - ), "StableAudioPipeline": ( "stable_audio", "pipeline_stable_audio", @@ -449,7 +429,7 @@ def _apply_sequence_parallel_if_enabled(model, od_config: OmniDiffusionConfig) - for attr in transformer_attrs: if not hasattr(model, attr): - # Some pipeline like LTX2TwoStagesPipeline have recursive + # Some pipelines have recursive # modules that have the transformer module = find_module_with_attr(model, attr) if module is None: @@ -519,13 +499,9 @@ _DIFFUSION_POST_PROCESS_FUNCS = { "WanPipeline": "get_wan22_post_process_func", "WanVACEPipeline": "get_wan22_vace_post_process_func", "LTX2Pipeline": "get_ltx2_post_process_func", - "LTX2TwoStagesPipeline": "get_ltx2_post_process_func", - "LTX2ImageToVideoPipeline": "get_ltx2_post_process_func", - "LTX2ImageToVideoTwoStagesPipeline": "get_ltx2_post_process_func", + "LTX2DistilledPipeline": "get_ltx2_post_process_func", "LTX2T2VDMD2Pipeline": "get_ltx2_post_process_func", "LTX2I2VDMD2Pipeline": "get_ltx2_post_process_func", - "LTX23Pipeline": "get_ltx2_post_process_func", - "LTX23ImageToVideoPipeline": "get_ltx2_post_process_func", "StableAudioPipeline": "get_stable_audio_post_process_func", "SoulXSingerPipeline": "get_soulxsinger_post_process_func", "SoulXSingerSVCPipeline": "get_soulxsinger_post_process_func", diff --git a/vllm_omni/diffusion/worker/request_batch.py b/vllm_omni/diffusion/worker/request_batch.py index 488d1eec2..079a213db 100644 --- a/vllm_omni/diffusion/worker/request_batch.py +++ b/vllm_omni/diffusion/worker/request_batch.py @@ -8,6 +8,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from typing import Any +import numpy as np import torch from vllm_omni.diffusion.data import DiffusionOutput @@ -20,7 +21,7 @@ def _slice_request_output(value: Any, start: int, stop: int) -> Any: return tuple(_slice_request_output(item, start, stop) for item in value) if isinstance(value, list): return value[start:stop] - if isinstance(value, torch.Tensor): + if isinstance(value, (np.ndarray, torch.Tensor)): return value[start:stop] return value diff --git a/vllm_omni/entrypoints/cli/serve.py b/vllm_omni/entrypoints/cli/serve.py index faec5f0b1..13ee808be 100644 --- a/vllm_omni/entrypoints/cli/serve.py +++ b/vllm_omni/entrypoints/cli/serve.py @@ -410,7 +410,7 @@ class OmniServeCommand(CLISubcommand): dest="model_class_name", type=str, default=None, - help="Override the diffusion pipeline class name (e.g. LTX2ImageToVideoPipeline).", + help="Override the diffusion pipeline class name (e.g. LTX2Pipeline).", ) omni_config_group.add_argument( "--diffusion-load-format", diff --git a/vllm_omni/model_extras/ltx2.py b/vllm_omni/model_extras/ltx2.py new file mode 100644 index 000000000..5bac58469 --- /dev/null +++ b/vllm_omni/model_extras/ltx2.py @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Request parameters shared by official LTX guidance pipelines.""" + +LTX_EXTRA_BODY_PARAMS = frozenset( + { + "video_cfg_scale", + "audio_cfg_scale", + "video_cfg_guidance_scale", + "audio_cfg_guidance_scale", + "video_stg_scale", + "audio_stg_scale", + "video_stg_guidance_scale", + "audio_stg_guidance_scale", + "video_modality_scale", + "audio_modality_scale", + "a2v_guidance_scale", + "v2a_guidance_scale", + "video_rescale_scale", + "audio_rescale_scale", + "video_stg_blocks", + "audio_stg_blocks", + } +) + +LTX_EXTRA_OUTPUT_PARAMS = frozenset() diff --git a/vllm_omni/model_extras/registry.py b/vllm_omni/model_extras/registry.py index edf28101c..dab5b45cf 100644 --- a/vllm_omni/model_extras/registry.py +++ b/vllm_omni/model_extras/registry.py @@ -37,6 +37,7 @@ from vllm_omni.model_extras.helios import ( ) from vllm_omni.model_extras.hunyuan_image3 import build_x_to_text_prompt as build_hunyuan_x_to_text_prompt from vllm_omni.model_extras.lingbot_video import LINGBOT_VIDEO_EXTRA_BODY_PARAMS +from vllm_omni.model_extras.ltx2 import LTX_EXTRA_BODY_PARAMS, LTX_EXTRA_OUTPUT_PARAMS from vllm_omni.model_extras.magi_human import ( MAGI_HUMAN_EXTRA_BODY_PARAMS, MAGI_HUMAN_EXTRA_OUTPUT_PARAMS, @@ -223,6 +224,16 @@ _EXTRA_SPECS: dict[str, dict[str, Any]] = { "LingBotVideoPipeline": { "extra_body_params": LINGBOT_VIDEO_EXTRA_BODY_PARAMS, }, + **{ + model_class_name: { + "extra_body_params": LTX_EXTRA_BODY_PARAMS, + "extra_output_params": LTX_EXTRA_OUTPUT_PARAMS, + } + for model_class_name in ( + "LTX2Pipeline", + "LTX2DistilledPipeline", + ) + }, "WanVACEPipeline": { "extra_body_params": VACE_EXTRA_BODY_PARAMS, "extra_output_params": VACE_EXTRA_OUTPUT_PARAMS, From f4640d9a2e97b311e0c8c3ae4ba793d9454f7cfa Mon Sep 17 00:00:00 2001 From: Ting FU Date: Tue, 28 Jul 2026 20:13:33 +0800 Subject: [PATCH 04/37] [BugFix] Fix 'GPUARWorker' attribute error (#5478) Signed-off-by: Semmer --- vllm_omni/worker/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vllm_omni/worker/base.py b/vllm_omni/worker/base.py index dd431d16b..4433e3d02 100644 --- a/vllm_omni/worker/base.py +++ b/vllm_omni/worker/base.py @@ -120,6 +120,7 @@ class OmniGPUWorkerBase(GPUWorker): self.model_runner.profile_run() self.non_torch_memory = profile_result.non_torch_increase + self.total_consumed = profile_result.total_consumed self.peak_activation_memory = profile_result.torch_peak_increase process_memory = ( From c96f0bc773ba0cc2e41f7678b2e1b8a1a891aff3 Mon Sep 17 00:00:00 2001 From: herotai214 <68222888+herotai214@users.noreply.github.com> Date: Tue, 28 Jul 2026 20:34:35 +0800 Subject: [PATCH 05/37] [Perf][GLM-Image] Skip redundant hidden state D2H (#5346) Signed-off-by: herotai214 --- tests/model_executor/models/glm_image/test_glm_image_ar.py | 4 ++++ vllm_omni/model_executor/models/glm_image/glm_image_ar.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/tests/model_executor/models/glm_image/test_glm_image_ar.py b/tests/model_executor/models/glm_image/test_glm_image_ar.py index 32a016b2a..5ccc9fc62 100644 --- a/tests/model_executor/models/glm_image/test_glm_image_ar.py +++ b/tests/model_executor/models/glm_image/test_glm_image_ar.py @@ -91,6 +91,10 @@ GlmImageRotaryEmbedding = _ar_mod.GlmImageRotaryEmbedding pytestmark = [pytest.mark.core_model, pytest.mark.cpu] +def test_glm_image_ar_omits_hidden_pooler_payload(): + assert GlmImageForConditionalGeneration.omni_pooler_payload_include_hidden is False + + # ============================================================================= # Helper: Minimal config for testing # ============================================================================= diff --git a/vllm_omni/model_executor/models/glm_image/glm_image_ar.py b/vllm_omni/model_executor/models/glm_image/glm_image_ar.py index 278e372d7..77aef398b 100644 --- a/vllm_omni/model_executor/models/glm_image/glm_image_ar.py +++ b/vllm_omni/model_executor/models/glm_image/glm_image_ar.py @@ -2414,6 +2414,11 @@ class GlmImageForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP # linear increments during decode. precomputed_mrope_decode = True + # The downstream GLM-Image diffusion stage consumes generated token ids and + # optional VQ-VAE image ids, not AR hidden states. Avoid copying one hidden + # vector to host at every decode step. + omni_pooler_payload_include_hidden = False + packed_modules_mapping = { "qkv_proj": [ "q_proj", From f775e8073219f30de1f359a9312b63ab91d90ef9 Mon Sep 17 00:00:00 2001 From: Hongsheng Liu Date: Tue, 28 Jul 2026 21:00:49 +0800 Subject: [PATCH 06/37] Revert "[BugFix] Fix 'GPUARWorker' attribute error" (#5507) --- vllm_omni/worker/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vllm_omni/worker/base.py b/vllm_omni/worker/base.py index 4433e3d02..dd431d16b 100644 --- a/vllm_omni/worker/base.py +++ b/vllm_omni/worker/base.py @@ -120,7 +120,6 @@ class OmniGPUWorkerBase(GPUWorker): self.model_runner.profile_run() self.non_torch_memory = profile_result.non_torch_increase - self.total_consumed = profile_result.total_consumed self.peak_activation_memory = profile_result.torch_peak_increase process_memory = ( From 61324e65d7f5574c183d331f6e30ad0f423a816b Mon Sep 17 00:00:00 2001 From: Canlin Guo Date: Tue, 28 Jul 2026 21:38:24 +0800 Subject: [PATCH 07/37] [Refactor] Make MOSS-TTS-Local stream session dynamic batch (#5235) Signed-off-by: gcanlin Signed-off-by: Canlin Guo --- vllm_omni/config/model.py | 1 + vllm_omni/config/stage_config.py | 4 + .../core/sched/omni_generation_scheduler.py | 18 +- vllm_omni/deploy/moss_tts_local.yaml | 20 +- .../chunk_transfer_adapter.py | 5 + vllm_omni/engine/arg_utils.py | 2 + .../models/moss_tts/audio_tokenizer_v2.py | 407 ++++++++++++++++-- .../cuda_graph_streaming_decoder_wrapper.py | 137 +++--- .../moss_tts/modeling_moss_tts_codec.py | 311 ++++++------- .../models/moss_tts/pipeline.py | 3 + .../stage_input_processors/moss_tts.py | 2 - 11 files changed, 616 insertions(+), 294 deletions(-) diff --git a/vllm_omni/config/model.py b/vllm_omni/config/model.py index 3da7de7d4..683c6bd52 100644 --- a/vllm_omni/config/model.py +++ b/vllm_omni/config/model.py @@ -119,6 +119,7 @@ class OmniModelConfig(ModelConfig): stage_id: int = 0 async_chunk: bool = False + retains_state_across_chunks: bool = False # Stage-1 active stream slots; 0 keeps legacy chunk-level round-robin. active_stream_window: int = 0 model_stage: str = "thinker" diff --git a/vllm_omni/config/stage_config.py b/vllm_omni/config/stage_config.py index 595003ec7..43944084c 100644 --- a/vllm_omni/config/stage_config.py +++ b/vllm_omni/config/stage_config.py @@ -223,6 +223,9 @@ class StagePipelineConfig: hf_config_name: str | None = None engine_output_type: str | None = None model_arch: str | None = None + # The model keeps per-request execution state while awaiting the next + # async chunk, so the parked request continues to consume model capacity. + retains_state_across_chunks: bool = False sampling_constraints: dict[str, Any] = field(default_factory=dict) custom_process_input_func: str | None = None custom_process_next_stage_input_func: str | None = None @@ -822,6 +825,7 @@ def _build_engine_args( ``engine_extras`` can still carry a stage-specific ``dtype``). """ engine_args: dict[str, Any] = {"model_arch": ps.model_arch or pipeline.model_arch or None} + engine_args["retains_state_across_chunks"] = ps.retains_state_across_chunks if ps.execution_type == StageExecutionType.DIFFUSION and ps.model_arch: engine_args.setdefault("model_class_name", ps.model_arch) if ps.engine_output_type: diff --git a/vllm_omni/core/sched/omni_generation_scheduler.py b/vllm_omni/core/sched/omni_generation_scheduler.py index 06082b0dc..26e2cfb83 100644 --- a/vllm_omni/core/sched/omni_generation_scheduler.py +++ b/vllm_omni/core/sched/omni_generation_scheduler.py @@ -47,6 +47,7 @@ class OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler): self.chunk_transfer_adapter = None if getattr(model_config, "async_chunk", False): self.chunk_transfer_adapter = OmniChunkTransferAdapter(self.vllm_config) + self._retains_state_across_chunks = bool(getattr(model_config, "retains_state_across_chunks", False)) self._pending_finish_reqs: list[Request] = [] self.input_coordinator: OmniSchedulingCoordinator | None = None if uses_full_payload_input_coordinator(model_config): @@ -145,12 +146,17 @@ class OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler): # Fast path selection and scheduling for one-shot generation requests, # independent of pooling_params. - while ( - self.waiting - and token_budget > 0 - and len(self.running) < self.max_num_running_reqs - and self._pause_state == PauseState.UNPAUSED - ): + while self.waiting and token_budget > 0 and self._pause_state == PauseState.UNPAUSED: + # Requests waiting for their next chunk are temporarily absent + # from `running`, but stateful models still retain their model + # runner slot. Mirror vLLM's treatment of + # `num_waiting_for_streaming_input` when enforcing max_num_seqs. + num_running = len(self.running) + if self._retains_state_across_chunks and self.chunk_transfer_adapter is not None: + num_running += self.chunk_transfer_adapter.num_running_waiting_for_chunk + if num_running >= self.max_num_running_reqs: + break + request = self.waiting.peek_request() # OMNI: Skip requests that are not in self.requests if request.request_id not in self.requests or ( diff --git a/vllm_omni/deploy/moss_tts_local.yaml b/vllm_omni/deploy/moss_tts_local.yaml index deffe4a17..7d3ae8601 100644 --- a/vllm_omni/deploy/moss_tts_local.yaml +++ b/vllm_omni/deploy/moss_tts_local.yaml @@ -9,24 +9,25 @@ connectors: shm: name: SharedMemoryConnector extra: - codec_streaming: true connector_get_sleep_s: 0.005 connector_get_max_wait_first_chunk: 1000 connector_get_max_wait: 300 initial_codec_chunk_frames: 1 # 1 frame * 80 ms/frame = 80 ms first audio chunk codec_chunk_frames: 15 # 15 frames * 80 ms/frame = 1.2 s steady chunks codec_left_context_frames: 0 - codec_stream_slots: 8 stages: - stage_id: 0 - max_num_seqs: 8 + max_num_seqs: 64 gpu_memory_utilization: 0.60 enable_prefix_caching: false async_scheduling: true max_num_batched_tokens: 65536 max_model_len: 4096 devices: "0" + # profiler_config: + # profiler: torch + # torch_profiler_dir: /root/vllm-omni-workspace/vllm-omni/results/moss_local_stage0/batch_8 output_connectors: to_stage_1: shm default_sampling_params: @@ -38,17 +39,22 @@ stages: repetition_penalty: 1.1 - stage_id: 1 - max_num_seqs: 8 - gpu_memory_utilization: 0.15 + max_num_seqs: 64 + gpu_memory_utilization: 0.30 enforce_eager: false enable_prefix_caching: false async_scheduling: true compilation_config: - cudagraph_capture_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + # Reused by the codec as execution-batch buckets; T is derived from + # initial_codec_chunk_frames and codec_chunk_frames. + cudagraph_capture_sizes: [1, 2, 4, 8, 16, 32, 64] cudagraph_num_of_warmups: 1 + # profiler_config: + # profiler: torch + # torch_profiler_dir: /root/vllm-omni-workspace/vllm-omni/results/moss_local_stage1/batch_8 max_num_batched_tokens: 65536 max_model_len: 65536 - devices: "0" + devices: "1" input_connectors: from_stage_0: shm default_sampling_params: diff --git a/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py b/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py index 83def2dc0..54d1477a9 100644 --- a/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py +++ b/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py @@ -550,6 +550,11 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): self._active_streams[request_id] = request return True + @property + def num_running_waiting_for_chunk(self) -> int: + """Count running requests temporarily removed while awaiting a chunk.""" + return len(self.waiting_for_chunk_running_requests) + def _preempt_non_active_running(self, waiting_queue: Any, running_queue: list[Request]) -> None: # Hold non-active running requests in a private deque rather than # routing them back through waiting_queue. Routing through the diff --git a/vllm_omni/engine/arg_utils.py b/vllm_omni/engine/arg_utils.py index 4d7622b91..924f41c3f 100644 --- a/vllm_omni/engine/arg_utils.py +++ b/vllm_omni/engine/arg_utils.py @@ -159,6 +159,7 @@ class OmniEngineArgs(EngineArgs): stage_connector_spec: dict[str, Any] = field(default_factory=dict) subtalker_sampling_params: dict[str, Any] | None = None async_chunk: bool = False + retains_state_across_chunks: bool = False # WS-A: Stage-1 active stream slots. 0 = legacy preempt-everything. # Must be declared here so engine_args dict propagation does not silently # drop the value when constructing OmniEngineArgs from kwargs. @@ -341,6 +342,7 @@ class OmniEngineArgs(EngineArgs): # All kwargs below are Omni specific stage_id=self.stage_id, async_chunk=self.async_chunk, + retains_state_across_chunks=self.retains_state_across_chunks, active_stream_window=self.active_stream_window, model_stage=self.model_stage, model_arch=self.model_arch, diff --git a/vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py b/vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py index f3ba18c36..26ea9c567 100644 --- a/vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py +++ b/vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py @@ -102,12 +102,22 @@ class StreamingState: def __post_init__(self): self.exec_mask = torch.ones(self.batch_size, dtype=torch.bool, device=self.device) + self._owns_exec_mask = True + + def bind_exec_mask(self, exec_mask: torch.Tensor) -> None: + if exec_mask.shape != (self.batch_size,): + raise ValueError(f"Expected exec_mask shape ({self.batch_size},), got {tuple(exec_mask.shape)}") + if exec_mask.device != self.device: + raise ValueError(f"Expected exec_mask on {self.device}, got {exec_mask.device}") + self.exec_mask = exec_mask + self._owns_exec_mask = False def set_exec_mask(self, exec_mask: torch.Tensor): self.exec_mask[:] = exec_mask def reset(self, reset_mask: torch.Tensor) -> None: - self.exec_mask[:] = torch.where(reset_mask, torch.ones_like(self.exec_mask), self.exec_mask) + if self._owns_exec_mask: + self.exec_mask.masked_fill_(reset_mask, True) def __enter__(self): # ExitStack expects a context manager; returning self is conventional and useful for debugging. @@ -117,6 +127,28 @@ class StreamingState: pass +@dataclass(frozen=True) +class StreamingExecutionContext: + """Map compact execution rows to persistent streaming-state slots. + + ``state_slot_ids`` and ``valid_rows`` both have shape ``(B_execution,)``. + CUDA Graph padding rows use dedicated scratch slots and ``valid_rows=False``. + """ + + state_slot_ids: torch.Tensor + valid_rows: torch.Tensor + + def validate(self, *, batch_size: int, state_capacity: int, device: torch.device) -> None: + if self.state_slot_ids.shape != (batch_size,): + raise ValueError(f"Expected state_slot_ids shape ({batch_size},), got {tuple(self.state_slot_ids.shape)}") + if self.valid_rows.shape != (batch_size,): + raise ValueError(f"Expected valid_rows shape ({batch_size},), got {tuple(self.valid_rows.shape)}") + if self.state_slot_ids.device != device or self.valid_rows.device != device: + raise ValueError("Streaming execution metadata must be on the same device as decoder inputs.") + if self.state_slot_ids.dtype != torch.long or self.valid_rows.dtype != torch.bool: + raise TypeError("state_slot_ids must be int64 and valid_rows must be bool.") + + class StreamingModule(nn.Module): """Base class for streaming components.""" @@ -447,11 +479,57 @@ class RingKVCache: def reset(self, reset_mask: torch.Tensor) -> None: self.end_offset[:] = torch.where(reset_mask, torch.zeros_like(self.end_offset), self.end_offset) - def complete(self, k: torch.Tensor, v: torch.Tensor, exec_mask: torch.Tensor) -> KVCacheResult: + def reset_slots(self, state_slot_ids: torch.Tensor) -> None: + if state_slot_ids.numel() == 0: + return + slots = state_slot_ids.to(device=self.end_offset.device, dtype=torch.long) + self.end_offset.index_fill_(0, slots, 0) + self.cache.index_fill_(1, slots, 0) + + def complete( + self, + k: torch.Tensor, + v: torch.Tensor, + exec_mask: torch.Tensor | None = None, + execution_context: StreamingExecutionContext | None = None, + ) -> KVCacheResult: B, H, T, D = k.shape if T <= 0: raise ValueError(f"Expected T > 0, got T={T}") + if execution_context is not None: + if not self.respect_exec_mask: + raise RuntimeError("Dynamic state-slot execution does not support weights_per_step attention.") + slots = execution_context.state_slot_ids + valid_rows = execution_context.valid_rows + end_offset = self.end_offset.index_select(0, slots) + row_cache = self.cache.index_select(1, slots) + + indexes = torch.arange(T, device=end_offset.device, dtype=end_offset.dtype) + indexes = (indexes + end_offset.view(-1, 1)) % self.capacity + scatter_indexes = indexes.view(B, 1, T, 1).expand(-1, H, T, D) + row_cache[0].scatter_(2, scatter_indexes, k) + row_cache[1].scatter_(2, scatter_indexes, v) + # Live and graph-padding rows always map to distinct slots. The + # latter map only to scratch state, so this write cannot corrupt a + # request even though dense graph operators still execute it. + self.cache.index_copy_(1, slots, row_cache) + + cache_indexes = torch.arange(self.capacity, device=end_offset.device, dtype=torch.long) + last_offset = end_offset.view(-1, 1) + T - 1 + end_index = last_offset % self.capacity + delta = cache_indexes - end_index + positions = torch.where( + delta <= 0, + last_offset + delta, + last_offset + delta - self.capacity, + ) + next_offset = torch.where(valid_rows, end_offset + T, end_offset) + self.end_offset.index_copy_(0, slots, next_offset) + invalid = cache_indexes >= next_offset.view(-1, 1) + positions = torch.where(invalid, torch.full_like(positions, -1), positions) + return KVCacheResult(row_cache[0], row_cache[1], positions) + indexes = torch.arange(T, device=self.end_offset.device, dtype=self.end_offset.dtype) indexes = indexes + self.end_offset.view(-1, 1) indexes = indexes % self.capacity @@ -479,6 +557,8 @@ class RingKVCache: ) if self.respect_exec_mask: + if exec_mask is None: + raise ValueError("exec_mask is required for fixed-width streaming execution.") self.end_offset[:] = torch.where(exec_mask, self.end_offset + T, self.end_offset) else: self.end_offset.add_(T) @@ -507,6 +587,14 @@ class MHAState(StreamingState): self.kv_cache.reset(reset_mask) self.offset_cpu = 0 + def reset_slots(self, state_slot_ids: torch.Tensor) -> None: + if state_slot_ids.numel() == 0: + return + slots = state_slot_ids.to(device=self.offset.device, dtype=torch.long) + self.offset.index_fill_(0, slots, 0) + if self.kv_cache is not None: + self.kv_cache.reset_slots(slots) + def apply_weights_per_step( modules: nn.ModuleList, @@ -600,7 +688,8 @@ class MossAudioTokenizerMultiheadAttention(StreamingModule): def _init_streaming_state(self, batch_size: int) -> MHAState: in_proj = cast(nn.Linear, self.in_projs[0]) device = cast(torch.device, in_proj.weight.device) - dtype = cast(torch.dtype, in_proj.weight.dtype) + weight_dtype = cast(torch.dtype, in_proj.weight.dtype) + dtype = torch.bfloat16 if device.type == "cuda" else weight_dtype dim_per_head = self.embed_dim // self.num_heads if self.context is None: @@ -625,21 +714,37 @@ class MossAudioTokenizerMultiheadAttention(StreamingModule): offset_cpu=0, ) - def _complete_kv(self, k, v) -> KVCacheResult: + def _complete_kv( + self, + k: torch.Tensor, + v: torch.Tensor, + execution_context: StreamingExecutionContext | None = None, + ) -> KVCacheResult: state = cast(MHAState | None, self._streaming_state) if state is None: return KVCacheResult.from_kv(k, v) if state.kv_cache is None: return KVCacheResult.from_kv(k, v) - return state.kv_cache.complete(k, v, state.exec_mask) + return state.kv_cache.complete(k, v, state.exec_mask, execution_context) - def forward(self, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor): + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + execution_context: StreamingExecutionContext | None = None, + ): state = cast(MHAState | None, self._streaming_state) B, T = query.shape[:2] if state is None: offset = torch.zeros(B, device=query.device, dtype=torch.long) offset_cpu = 0 + elif execution_context is not None: + if self.weights_per_step: + raise RuntimeError("Dynamic codec state slots do not support weights_per_step attention.") + offset = state.offset.index_select(0, execution_context.state_slot_ids) + offset_cpu = 0 else: offset = state.offset offset_cpu = state.offset_cpu @@ -652,7 +757,7 @@ class MossAudioTokenizerMultiheadAttention(StreamingModule): if self.rope: q, k = self.rope(q, k, offset, time_before_heads=False) - k, v, pos_k = self._complete_kv(k, v) + k, v, pos_k = self._complete_kv(k, v, execution_context) pos_k = pos_k[:, None] if self.causal: @@ -670,8 +775,12 @@ class MossAudioTokenizerMultiheadAttention(StreamingModule): x = apply_weights_per_step(self.out_projs, self.weights_per_step_schedule, x, offset_cpu) if state is not None: - state.offset[:] = torch.where(state.exec_mask, state.offset + T, state.offset) - state.offset_cpu += T + if execution_context is None: + state.offset[:] = torch.where(state.exec_mask, state.offset + T, state.offset) + state.offset_cpu += T + else: + next_offset = torch.where(execution_context.valid_rows, offset + T, offset) + state.offset.index_copy_(0, execution_context.state_slot_ids, next_offset) return x @@ -688,6 +797,11 @@ class LayerState(StreamingState): super().reset(reset_mask) self.offset_cpu = 0 + def reset_slots(self, state_slot_ids: torch.Tensor) -> None: + # This state is only meaningful for weights_per_step models, which are + # rejected by the dynamic state-slot path. + self.offset_cpu = 0 + class MossAudioTokenizerTransformerLayer(StreamingModule): """Transformer layer with streaming support.""" @@ -763,8 +877,14 @@ class MossAudioTokenizerTransformerLayer(StreamingModule): device = next(iter(self.parameters())).device return LayerState(batch_size, device, offset_cpu=0) - def _ff_block(self, x: torch.Tensor) -> torch.Tensor: + def _ff_block( + self, + x: torch.Tensor, + execution_context: StreamingExecutionContext | None = None, + ) -> torch.Tensor: state = self._streaming_state + if execution_context is not None and self.weights_per_step: + raise RuntimeError("Dynamic codec state slots do not support weights_per_step transformer layers.") offset = state.offset_cpu if isinstance(state, LayerState) else 0 x_orig = x @@ -782,17 +902,25 @@ class MossAudioTokenizerTransformerLayer(StreamingModule): update = self.gating(x) return x_orig.to(update) + self.layer_scale_2(update) - def _sa_block(self, x: torch.Tensor): + def _sa_block( + self, + x: torch.Tensor, + execution_context: StreamingExecutionContext | None = None, + ): x_orig = x x = self.norm1(x) - update = self.self_attn(x, x, x) + update = self.self_attn(x, x, x, execution_context=execution_context) return x_orig.to(update) + self.layer_scale_1(update) - def forward(self, x: torch.Tensor): - x = self._sa_block(x) - x = self._ff_block(x) + def forward( + self, + x: torch.Tensor, + execution_context: StreamingExecutionContext | None = None, + ): + x = self._sa_block(x, execution_context) + x = self._ff_block(x, execution_context) state = self._streaming_state - if state is not None: + if state is not None and execution_context is None: assert isinstance(state, LayerState) state.offset_cpu += x.shape[1] return x @@ -811,6 +939,11 @@ class TransformerState(StreamingState): super().reset(reset_mask) self.offsets[:] = torch.where(reset_mask, torch.zeros_like(self.offsets), self.offsets) + def reset_slots(self, state_slot_ids: torch.Tensor) -> None: + if state_slot_ids.numel() == 0: + return + self.offsets.index_fill_(0, state_slot_ids.to(device=self.offsets.device, dtype=torch.long), 0) + class MossAudioTokenizerTransformer(StreamingModule): """Transformer with streaming/causal support.""" @@ -869,15 +1002,17 @@ class MossAudioTokenizerTransformer(StreamingModule): def forward(self, x: torch.Tensor, *args, **kwargs): B, T, C = x.shape state = self._streaming_state - offsets = ( - torch.zeros(1, dtype=torch.long, device=x.device) - if state is None - else ( - state.offsets - if isinstance(state, TransformerState) - else torch.zeros(1, dtype=torch.long, device=x.device) - ) - ) + execution_context = kwargs.get("execution_context") + if execution_context is not None and not isinstance(execution_context, StreamingExecutionContext): + raise TypeError("execution_context must be a StreamingExecutionContext.") + if state is None: + offsets = torch.zeros(1, dtype=torch.long, device=x.device) + elif not isinstance(state, TransformerState): + raise TypeError(f"Unexpected transformer state: {type(state).__name__}") + elif execution_context is None: + offsets = state.offsets + else: + offsets = state.offsets.index_select(0, execution_context.state_slot_ids) if self.positional_embedding in {"sin", "sin_rope"}: positions = torch.arange(T, device=x.device).view(1, -1, 1) @@ -890,7 +1025,11 @@ class MossAudioTokenizerTransformer(StreamingModule): if state is not None: assert isinstance(state, TransformerState) - state.offsets[:] = torch.where(state.exec_mask, state.offsets + T, state.offsets) + if execution_context is None: + state.offsets[:] = torch.where(state.exec_mask, state.offsets + T, state.offsets) + else: + next_offsets = torch.where(execution_context.valid_rows, offsets + T, offsets) + state.offsets.index_copy_(0, execution_context.state_slot_ids, next_offsets) return x @@ -1235,6 +1374,76 @@ class MossAudioTokenizerResidualLFQ(nn.Module): for _ in range(num_quantizers) ] ) + self.register_buffer("_decode_lut", None, persistent=False) + self.register_buffer("_decode_lut_bias", None, persistent=False) + self.register_buffer("_decode_lut_quantizer_ids", None, persistent=False) + + @torch.no_grad() + def build_decode_lut( + self, + num_quantizers: int, + *, + dtype: torch.dtype, + ) -> None: + """Fold the inference-only LFQ decode projections into code tables.""" + if not 0 < num_quantizers <= len(self.quantizers): + raise ValueError(f"num_quantizers must be in [1, {len(self.quantizers)}], got {num_quantizers}") + + output_proj = self.output_proj + if isinstance(output_proj, nn.Identity): + output_weight = None + output_bias = torch.zeros( + self.output_dim, + device=self.quantizers[0].codebook.weight.device, + dtype=torch.float32, + ) + elif isinstance(output_proj, nn.Conv1d) and output_proj.kernel_size == (1,): + output_weight = output_proj.weight.detach().squeeze(-1).float() + output_bias = ( + output_proj.bias.detach().float() + if output_proj.bias is not None + else torch.zeros( + self.output_dim, + device=output_weight.device, + dtype=torch.float32, + ) + ) + else: + raise TypeError( + "LFQ decoded LUT requires output_proj to be Identity or Conv1d(kernel_size=1), " + f"got {type(output_proj).__name__}" + ) + + decoded_tables: list[torch.Tensor] = [] + for quantizer in self.quantizers[:num_quantizers]: + quantizer = cast(MossAudioTokenizerLFQ, quantizer) + inner_proj = quantizer.out_proj + codebook = quantizer.codebook.weight.detach().float() + if isinstance(inner_proj, nn.Identity): + decoded = codebook + elif isinstance(inner_proj, nn.Conv1d) and inner_proj.kernel_size == (1,): + inner_weight = inner_proj.weight.detach().squeeze(-1).float() + inner_bias = inner_proj.bias.detach().float() if inner_proj.bias is not None else None + decoded = F.linear(codebook, inner_weight, inner_bias) + else: + raise TypeError( + "LFQ decoded LUT requires quantizer out_proj to be Identity or Conv1d(kernel_size=1), " + f"got {type(inner_proj).__name__}" + ) + + if output_weight is not None: + # Apply the shared outer bias once after summing all codebooks. + decoded = F.linear(decoded, output_weight, bias=None) + decoded_tables.append(decoded) + + lut = torch.stack(decoded_tables).to(dtype=dtype) + self._decode_lut = lut + self._decode_lut_bias = output_bias.to(dtype=dtype) + self._decode_lut_quantizer_ids = torch.arange( + num_quantizers, + device=lut.device, + dtype=torch.long, + ).view(-1, 1, 1) @torch.no_grad() def forward( @@ -1276,6 +1485,12 @@ class MossAudioTokenizerResidualLFQ(nn.Module): def decode_codes(self, codes: torch.Tensor) -> torch.Tensor: nq, B, T = codes.shape + if self._decode_lut is not None and nq <= self._decode_lut.shape[0]: + quantizer_ids = self._decode_lut_quantizer_ids[:nq] + decoded = self._decode_lut[quantizer_ids, codes].sum(dim=0) + decoded = decoded + self._decode_lut_bias + return decoded.transpose(1, 2) + emb = torch.zeros(B, self.rvq_dim, T, device=codes.device, dtype=torch.float32) for i, quantizer in enumerate(self.quantizers[:nq]): quantizer = cast(MossAudioTokenizerLFQ, quantizer) @@ -1389,34 +1604,114 @@ class MossAudioTokenizerModel(MossAudioTokenizerPreTrainedModel): f"got current_frame_rate={current_frame_rate}, expected={expected_output_frame_rate}." ) + self._streaming_modules: list[StreamingModule] = [] + self._streaming_exec_mask: torch.Tensor | None = None + self._decoder_state_capacity = 0 self.post_init() - def _start_streaming(self, batch_size: int): - """Start streaming mode for all modules.""" + def _start_streaming(self, batch_size: int, *, decoder_only: bool = False) -> None: + """Start streaming with one shared, address-stable execution mask.""" + if self._streaming_modules: + raise RuntimeError("MOSS Audio Tokenizer is already streaming.") + + root: nn.Module = self.decoder if decoder_only else self + shared_exec_mask = torch.ones( + batch_size, + dtype=torch.bool, + device=next(self.parameters()).device, + ) + streaming_modules: list[StreamingModule] = [] def _start(module): if isinstance(module, StreamingModule): - module._streaming_state = module._init_streaming_state(batch_size) + if module._streaming_state is not None: + raise RuntimeError("A MOSS Audio Tokenizer submodule is already streaming.") + state = module._init_streaming_state(batch_size) + state.bind_exec_mask(shared_exec_mask) + module._streaming_state = state + streaming_modules.append(module) - self.apply(_start) + root.apply(_start) + self._streaming_modules = streaming_modules + self._streaming_exec_mask = shared_exec_mask - def _stop_streaming(self): + def _stop_streaming(self) -> None: """Stop streaming mode for all modules.""" + for module in self._streaming_modules: + module._streaming_state = None + self._streaming_modules = [] + self._streaming_exec_mask = None + self._decoder_state_capacity = 0 - def _stop(module): - if isinstance(module, StreamingModule): - module._streaming_state = None + def initialize_decoder_state_pool(self, state_capacity: int, scratch_capacity: int = 0) -> None: + """Allocate persistent decoder state independently of execution B.""" + if state_capacity <= 0 or scratch_capacity < 0: + raise ValueError(f"Invalid decoder state capacities: state={state_capacity}, scratch={scratch_capacity}.") + self._start_streaming(state_capacity + scratch_capacity, decoder_only=True) + self._decoder_state_capacity = state_capacity + scratch_capacity - self.apply(_stop) + def close_decoder_state_pool(self) -> None: + self._stop_streaming() + + def reset_decoder_state_slots(self, state_slot_ids: torch.Tensor) -> None: + if not self._streaming_modules: + raise RuntimeError("MOSS Audio Tokenizer decoder state pool is not initialized.") + if state_slot_ids.numel() == 0: + return + slots = state_slot_ids.to(device=next(self.parameters()).device, dtype=torch.long) + if not torch.cuda.is_current_stream_capturing(): + slots_cpu = slots.detach().to("cpu") + if int(slots_cpu.min()) < 0 or int(slots_cpu.max()) >= self._decoder_state_capacity: + raise ValueError( + f"Decoder state slots must be in [0, {self._decoder_state_capacity}), got {slots_cpu.tolist()}" + ) + for module in self._streaming_modules: + state = module._streaming_state + reset_slots = getattr(state, "reset_slots", None) + if callable(reset_slots): + reset_slots(slots) + + def decode_streaming_batch( + self, + codes: torch.Tensor, + codes_lengths: torch.Tensor, + state_slot_ids: torch.Tensor, + valid_rows: torch.Tensor, + ) -> MossAudioTokenizerDecoderOutput: + if not self._streaming_modules: + raise RuntimeError("MOSS Audio Tokenizer decoder state pool is not initialized.") + execution_context = StreamingExecutionContext(state_slot_ids=state_slot_ids, valid_rows=valid_rows) + execution_context.validate( + batch_size=int(codes.shape[1]), + state_capacity=self._decoder_state_capacity, + device=codes.device, + ) + return self._decode_frame(codes, codes_lengths, execution_context=execution_context) def _set_streaming_exec_mask(self, exec_mask: torch.Tensor) -> None: - """Set the active slot mask on every live streaming module.""" + """Update the shared active-slot mask with one device copy.""" + shared_exec_mask = self._streaming_exec_mask + if shared_exec_mask is None: + raise RuntimeError("MOSS Audio Tokenizer is not streaming.") + if exec_mask.shape != shared_exec_mask.shape: + raise ValueError(f"Expected exec_mask shape {tuple(shared_exec_mask.shape)}, got {tuple(exec_mask.shape)}") + shared_exec_mask.copy_(exec_mask.to(device=shared_exec_mask.device, dtype=torch.bool), non_blocking=True) - def _set(module): - if isinstance(module, StreamingModule) and module._streaming_state is not None: - module._streaming_state.set_exec_mask(exec_mask.to(module._streaming_state.device)) - - self.apply(_set) + def _reset_streaming_slots(self, reset_mask: torch.Tensor) -> None: + """Reset cached decoder state without traversing the full module tree.""" + shared_exec_mask = self._streaming_exec_mask + if shared_exec_mask is None: + raise RuntimeError("MOSS Audio Tokenizer is not streaming.") + if reset_mask.shape != shared_exec_mask.shape: + raise ValueError( + f"Expected reset_mask shape {tuple(shared_exec_mask.shape)}, got {tuple(reset_mask.shape)}" + ) + reset_mask = reset_mask.to(device=shared_exec_mask.device, dtype=torch.bool) + shared_exec_mask.masked_fill_(reset_mask, True) + for module in self._streaming_modules: + state = module._streaming_state + if state is not None: + state.reset(reset_mask) @contextmanager def streaming(self, batch_size: int = 1): @@ -1427,6 +1722,15 @@ class MossAudioTokenizerModel(MossAudioTokenizerPreTrainedModel): finally: self._stop_streaming() + @contextmanager + def decoder_streaming(self, batch_size: int = 1): + """Stream decoder-only inference without allocating encoder state.""" + self._start_streaming(batch_size, decoder_only=True) + try: + yield + finally: + self._stop_streaming() + def _prepare_waveform_batch(self, wav_list: list[torch.Tensor]) -> tuple[torch.Tensor, torch.Tensor]: if len(wav_list) == 0: raise ValueError("`wav_list` must contain at least one waveform.") @@ -1625,6 +1929,7 @@ class MossAudioTokenizerModel(MossAudioTokenizerPreTrainedModel): self, codes: torch.Tensor, codes_lengths: torch.Tensor | None = None, + execution_context: StreamingExecutionContext | None = None, ) -> MossAudioTokenizerDecoderOutput: """Detokenize discrete tokens into audio waveform.""" nq, B, T = codes.shape @@ -1633,13 +1938,21 @@ class MossAudioTokenizerModel(MossAudioTokenizerPreTrainedModel): if codes_lengths is None: codes_lengths = torch.full((B,), T, device=device, dtype=torch.long) - # Decode from codes - quantizer = cast(MossAudioTokenizerResidualVQ | MossAudioTokenizerResidualLFQ, self.quantizer) - zq = quantizer.decode_codes(codes) + # Keep eager execution and CUDA Graph capture on the same BF16 path. + with torch.autocast(device_type=device.type, dtype=torch.bfloat16, enabled=device.type == "cuda"): + quantizer = cast(MossAudioTokenizerResidualVQ | MossAudioTokenizerResidualLFQ, self.quantizer) + zq = quantizer.decode_codes(codes) - d, d_lengths = zq, codes_lengths - for decoder_module in self.decoder: - d, d_lengths = decoder_module(d, d_lengths) + d, d_lengths = zq, codes_lengths + for decoder_module in self.decoder: + if isinstance(decoder_module, StreamingModule): + d, d_lengths = decoder_module( + d, + d_lengths, + execution_context=execution_context, + ) + else: + d, d_lengths = decoder_module(d, d_lengths) d, d_lengths = self._restore_channels_from_codec(d, d_lengths) return MossAudioTokenizerDecoderOutput(audio=d, audio_lengths=d_lengths) diff --git a/vllm_omni/model_executor/models/moss_tts/cuda_graph_streaming_decoder_wrapper.py b/vllm_omni/model_executor/models/moss_tts/cuda_graph_streaming_decoder_wrapper.py index 1daa5081e..4aeb17785 100644 --- a/vllm_omni/model_executor/models/moss_tts/cuda_graph_streaming_decoder_wrapper.py +++ b/vllm_omni/model_executor/models/moss_tts/cuda_graph_streaming_decoder_wrapper.py @@ -1,12 +1,11 @@ # Copyright 2026 OpenMOSS and the vLLM-Omni team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). -"""CUDA graph wrapper for MOSS-TTS Local streaming codec decode.""" +"""CUDA graphs for compact MOSS-TTS streaming codec batches.""" from __future__ import annotations import time -from collections.abc import Callable from dataclasses import dataclass import torch @@ -23,31 +22,35 @@ class _CapturedStreamingDecodeGraph: graph: CUDAGraph static_codes: torch.Tensor static_lengths: torch.Tensor + static_state_slot_ids: torch.Tensor + static_valid_rows: torch.Tensor static_audio: torch.Tensor static_audio_lengths: torch.Tensor class CUDAGraphStreamingDecoderWrapper: - """Replay fixed-size streaming ``_decode_frame`` steps with CUDA graphs. + """Replay streaming decode graphs keyed by ``(B_bucket, exact_T)``. - The wrapped codec must already be inside ``codec.streaming(batch_size)``. - Graphs are keyed by the streaming step frame count ``T``. The batch width is - fixed to the codec stream slot count. + Persistent codec state is addressed through ``state_slot_ids``. Graph + padding rows use non-leaseable scratch slots, so graph batch width is no + longer tied to the number of live stream states. """ def __init__( self, codec: nn.Module, *, - batch_size: int, + state_capacity: int, + batch_sizes: list[int], + frame_sizes: list[int], num_quantizers: int, - reset_streaming_state: Callable[[], None] | None = None, ) -> None: self.codec = codec - self.batch_size = int(batch_size) + self.state_capacity = int(state_capacity) + self.batch_sizes = sorted({int(size) for size in batch_sizes if 0 < int(size) <= state_capacity}) + self.frame_sizes = sorted({int(size) for size in frame_sizes if int(size) > 0}) self.num_quantizers = int(num_quantizers) - self.reset_streaming_state = reset_streaming_state - self.graphs: dict[int, _CapturedStreamingDecodeGraph] = {} + self.graphs: dict[tuple[int, int], _CapturedStreamingDecodeGraph] = {} self._pool = None self._warmed_up = False @@ -56,109 +59,135 @@ class CUDAGraphStreamingDecoderWrapper: return bool(self.graphs) @property - def capture_sizes(self) -> list[int]: + def capture_sizes(self) -> list[tuple[int, int]]: return sorted(self.graphs) + @property + def scratch_capacity(self) -> int: + return max(self.batch_sizes, default=0) + @torch.no_grad() - def warmup(self, device: torch.device, capture_sizes: list[int]) -> None: + def warmup(self, device: torch.device) -> None: if self._warmed_up: return self._warmed_up = True - if device.type != "cuda": - return - sizes = sorted({int(size) for size in capture_sizes if int(size) > 0}, reverse=True) - if not sizes: + if device.type != "cuda" or not self.batch_sizes or not self.frame_sizes: return + capture_keys = sorted( + ((batch_size, frame_size) for batch_size in self.batch_sizes for frame_size in self.frame_sizes), + reverse=True, + ) logger.info( - "MOSS-TTS streaming decoder CUDA graph warmup: batch_size=%d n_vq=%d capture_sizes=%s", - self.batch_size, + "MOSS-TTS streaming decoder CUDA graph warmup: n_vq=%d (B,T)=%s", self.num_quantizers, - list(reversed(sizes)), + list(reversed(capture_keys)), ) start_s = time.perf_counter() with torch.cuda.device(device): - for size in sizes: + for batch_size, frame_size in capture_keys: + key = (batch_size, frame_size) try: - self._capture(size, device) - logger.info(" Captured MOSS-TTS streaming decoder CUDA graph for size=%d", size) + self._capture(batch_size, frame_size, device) + logger.info(" Captured MOSS-TTS streaming decoder CUDA graph for (B,T)=%s", key) except Exception: - self.graphs.pop(size, None) + self.graphs.pop(key, None) logger.warning( - " Failed to capture MOSS-TTS streaming decoder CUDA graph for size=%d; " - "this size will use eager decode", - size, + " Failed to capture MOSS-TTS streaming decoder CUDA graph for (B,T)=%s; using eager", + key, exc_info=True, ) logger.info( "MOSS-TTS streaming decoder CUDA graph warmup complete: %d/%d captured in %.1f ms", len(self.graphs), - len(sizes), + len(capture_keys), (time.perf_counter() - start_s) * 1000.0, ) @torch.no_grad() - def _capture(self, size: int, device: torch.device) -> None: + def _capture(self, batch_size: int, frame_size: int, device: torch.device) -> None: codes = torch.zeros( self.num_quantizers, - self.batch_size, - size, + batch_size, + frame_size, dtype=torch.long, device=device, ) - lengths = torch.full((self.batch_size,), size, dtype=torch.long, device=device) - exec_mask = torch.ones(self.batch_size, dtype=torch.bool, device=device) + lengths = torch.zeros(batch_size, dtype=torch.long, device=device) + scratch_slots = self.state_capacity + torch.arange(batch_size, dtype=torch.long, device=device) + valid_rows = torch.zeros(batch_size, dtype=torch.bool, device=device) - self.codec._set_streaming_exec_mask(exec_mask) stream = torch.cuda.Stream() stream.wait_stream(torch.cuda.current_stream()) with torch.cuda.stream(stream): - _ = self.codec._decode_frame(codes, lengths) + _ = self.codec.decode_streaming_batch(codes, lengths, scratch_slots, valid_rows) torch.cuda.current_stream().wait_stream(stream) torch.accelerator.synchronize(device) + self.codec.reset_decoder_state_slots(scratch_slots) - if self.reset_streaming_state is not None: - self.reset_streaming_state() - self.codec._set_streaming_exec_mask(exec_mask) if self._pool is None: self._pool = current_platform.get_global_graph_pool() graph = CUDAGraph() with torch.cuda.graph(graph, pool=self._pool, capture_error_mode="thread_local"): - output = self.codec._decode_frame(codes, lengths) + output = self.codec.decode_streaming_batch(codes, lengths, scratch_slots, valid_rows) if output.audio is None or output.audio_lengths is None: raise RuntimeError("MOSS-TTS streaming decoder CUDA graph capture produced empty audio.") - - self.graphs[size] = _CapturedStreamingDecodeGraph( + self.graphs[(batch_size, frame_size)] = _CapturedStreamingDecodeGraph( graph=graph, static_codes=codes, static_lengths=lengths, + static_state_slot_ids=scratch_slots, + static_valid_rows=valid_rows, static_audio=output.audio, static_audio_lengths=output.audio_lengths, ) + def _select_batch_size(self, actual_batch_size: int) -> int | None: + return next((size for size in self.batch_sizes if size >= actual_batch_size), None) + + def _select_frame_size(self, actual_frame_size: int, allow_padding: bool) -> int | None: + if actual_frame_size in self.frame_sizes: + return actual_frame_size + if not allow_padding: + return None + return next((size for size in self.frame_sizes if size >= actual_frame_size), None) + @torch.no_grad() def decode( self, - codes_step: torch.Tensor, - exec_mask: torch.Tensor, - ) -> tuple[torch.Tensor, torch.Tensor] | None: - if not codes_step.is_cuda or torch.cuda.is_current_stream_capturing(): + codes: torch.Tensor, + state_slot_ids: torch.Tensor, + *, + allow_frame_padding: bool = False, + ) -> tuple[torch.Tensor, torch.Tensor, int] | None: + if not codes.is_cuda or torch.cuda.is_current_stream_capturing(): return None - n_vq, batch_size, step_t = codes_step.shape - if int(n_vq) != self.num_quantizers or int(batch_size) != self.batch_size: + n_vq, actual_batch_size, frame_size = codes.shape + if int(n_vq) != self.num_quantizers or state_slot_ids.shape != (actual_batch_size,): return None - # The streaming codec advances its internal KV offsets by tensor T, so - # bucket-padding a smaller step into a larger captured graph would write - # padded frames into state. Capture exact step sizes instead. - entry = self.graphs.get(int(step_t)) + batch_size = self._select_batch_size(int(actual_batch_size)) + if batch_size is None: + return None + graph_frame_size = self._select_frame_size(int(frame_size), allow_frame_padding) + if graph_frame_size is None: + return None + entry = self.graphs.get((batch_size, graph_frame_size)) if entry is None: return None - self.codec._set_streaming_exec_mask(exec_mask) - entry.static_codes.copy_(codes_step) + entry.static_codes.zero_() + entry.static_codes[:, :actual_batch_size, :frame_size].copy_(codes, non_blocking=True) + entry.static_lengths.zero_() + entry.static_lengths[:actual_batch_size].fill_(int(frame_size)) + entry.static_state_slot_ids.copy_( + self.state_capacity + torch.arange(batch_size, dtype=torch.long, device=entry.static_state_slot_ids.device) + ) + entry.static_state_slot_ids[:actual_batch_size].copy_(state_slot_ids, non_blocking=True) + entry.static_valid_rows.zero_() + entry.static_valid_rows[:actual_batch_size].fill_(True) entry.graph.replay() - return entry.static_audio, entry.static_audio_lengths + return entry.static_audio, entry.static_audio_lengths, int(actual_batch_size) __all__ = ["CUDAGraphStreamingDecoderWrapper"] diff --git a/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py b/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py index d0f9ea1bc..d6bc525ac 100644 --- a/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py +++ b/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py @@ -5,7 +5,6 @@ from __future__ import annotations -import contextlib from collections.abc import Iterable from typing import Any @@ -38,104 +37,136 @@ logger = init_logger(__name__) class _MossCodecStreamSession: - """Persistent streaming decode session for vendored MOSS-Audio-Tokenizer-v2.""" + """Persistent state pool with compact per-step execution batches.""" def __init__( self, codec: nn.Module, *, - stream_slots: int, + state_capacity: int, n_vq: int, - cudagraph_capture_sizes: list[int] | None = None, + graph_batch_sizes: list[int] | None = None, + graph_frame_sizes: list[int] | None = None, ) -> None: self._codec = codec - self._stream_slots = int(stream_slots) - self._batch_size = self._stream_slots + self._state_capacity = int(state_capacity) self._n_vq = int(n_vq) self._device = next(codec.parameters()).device - self._free_stream_slots = list(range(self._stream_slots)) - self._exit_stack = contextlib.ExitStack() + self._free_stream_slots = list(reversed(range(self._state_capacity))) + self._leased_slots: set[int] = set() self._closed = False - with torch.no_grad(): - self._exit_stack.enter_context(codec.streaming(self._batch_size)) self._cudagraph_wrapper: CUDAGraphStreamingDecoderWrapper | None = None - capture_sizes = sorted({int(size) for size in (cudagraph_capture_sizes or []) if int(size) > 0}) - if capture_sizes and self._device.type == "cuda": + batch_sizes = sorted({int(size) for size in (graph_batch_sizes or []) if 0 < int(size) <= self._state_capacity}) + frame_sizes = sorted({int(size) for size in (graph_frame_sizes or []) if int(size) > 0}) + scratch_capacity = max(batch_sizes, default=0) if self._device.type == "cuda" else 0 + initialize_state_pool = getattr(codec, "initialize_decoder_state_pool", None) + if not callable(initialize_state_pool): + raise RuntimeError("The streaming codec does not implement a decoder state pool.") + with torch.no_grad(): + initialize_state_pool(self._state_capacity, scratch_capacity) + if batch_sizes and frame_sizes and self._device.type == "cuda": self._cudagraph_wrapper = CUDAGraphStreamingDecoderWrapper( codec, - batch_size=self._batch_size, + state_capacity=self._state_capacity, + batch_sizes=batch_sizes, + frame_sizes=frame_sizes, num_quantizers=self._n_vq, - reset_streaming_state=lambda: self.reset_slots(list(range(self._batch_size))), ) - self._cudagraph_wrapper.warmup(self._device, capture_sizes) - self.reset_slots(list(range(self._batch_size))) + self._cudagraph_wrapper.warmup(self._device) + self.reset_slots(list(range(self._state_capacity + scratch_capacity))) if not self._cudagraph_wrapper.is_ready: self._cudagraph_wrapper = None def acquire(self) -> int | None: if not self._free_stream_slots: return None - return self._free_stream_slots.pop() + slot = self._free_stream_slots.pop() + self._leased_slots.add(slot) + return slot def release(self, slot: int) -> None: if self._closed: return + if slot not in self._leased_slots: + return self.reset_slots([slot]) + self._leased_slots.remove(slot) self._free_stream_slots.append(slot) def reset_slots(self, slots: list[int]) -> None: if not slots: return - reset_mask = torch.zeros(self._batch_size, dtype=torch.bool, device=self._device) - reset_mask[slots] = True - - def _reset(module: nn.Module) -> None: - state = getattr(module, "_streaming_state", None) - if state is not None: - state.reset(reset_mask.to(state.device)) - - with torch.no_grad(): - self._codec.apply(_reset) + reset_streaming_slots = getattr(self._codec, "reset_decoder_state_slots", None) + if callable(reset_streaming_slots): + with torch.no_grad(): + reset_streaming_slots(torch.tensor(slots, device=self._device, dtype=torch.long)) + return + raise RuntimeError("The streaming codec does not implement state-slot reset.") def close(self) -> None: if self._closed: return + close_state_pool = getattr(self._codec, "close_decoder_state_pool", None) with torch.no_grad(): - self._exit_stack.close() + if callable(close_state_pool): + close_state_pool() self._closed = True @torch.no_grad() - def step(self, slot_codes: dict[int, torch.Tensor]) -> dict[int, torch.Tensor]: + def step( + self, + slot_codes: dict[int, torch.Tensor], + *, + terminal_slots: set[int] | None = None, + ) -> dict[int, torch.Tensor]: if not slot_codes: return {} + terminal_slots = terminal_slots or set() + if not terminal_slots.issubset(slot_codes): + raise ValueError("terminal_slots must be a subset of the decode batch slots.") step_lengths = {int(codes.shape[1]) for codes in slot_codes.values()} if len(step_lengths) != 1: raise ValueError(f"MOSS codec streaming step needs uniform T, got {sorted(step_lengths)}") (step_t,) = step_lengths - codes_step = torch.zeros( - self._n_vq, - self._batch_size, - step_t, - dtype=torch.long, - device=self._device, + slots = list(slot_codes) + if any(slot not in self._leased_slots for slot in slots): + raise RuntimeError(f"Streaming decode references an unleased state slot: {slots}") + codes_step = torch.stack( + [slot_codes[slot].to(device=self._device, dtype=torch.long) for slot in slots], + dim=1, ) - codes_lengths = torch.zeros(self._batch_size, dtype=torch.long, device=self._device) - exec_mask = torch.zeros(self._batch_size, dtype=torch.bool, device=self._device) - for slot, codes in slot_codes.items(): - codes_step[:, slot, :] = codes.to(device=self._device, dtype=torch.long) - codes_lengths[slot] = int(codes.shape[1]) - exec_mask[slot] = True + state_slot_ids = torch.tensor(slots, device=self._device, dtype=torch.long) - graph_output: tuple[torch.Tensor, torch.Tensor] | None = None + graph_output: tuple[torch.Tensor, torch.Tensor, int] | None = None if self._cudagraph_wrapper is not None: - graph_output = self._cudagraph_wrapper.decode(codes_step, exec_mask) + # A terminal tail can safely pad T to a larger graph: decoder + # attention is causal, audio_lengths trims the padding, and every + # affected state slot is reset immediately after this step. + graph_output = self._cudagraph_wrapper.decode( + codes_step, + state_slot_ids, + allow_frame_padding=len(terminal_slots) == len(slots), + ) used_cudagraph = graph_output is not None if used_cudagraph: - audio_tensor, lengths_tensor = graph_output + audio_tensor, lengths_tensor, actual_batch_size = graph_output + audio_tensor = audio_tensor[:actual_batch_size] + lengths_tensor = lengths_tensor[:actual_batch_size] else: - self._codec._set_streaming_exec_mask(exec_mask) - result = self._codec._decode_frame(codes_step, codes_lengths) + codes_lengths = torch.full( + (len(slots),), + step_t, + dtype=torch.long, + device=self._device, + ) + valid_rows = torch.ones(len(slots), dtype=torch.bool, device=self._device) + result = self._codec.decode_streaming_batch( + codes_step, + codes_lengths, + state_slot_ids, + valid_rows, + ) if result.audio is None: return {} audio_tensor = result.audio @@ -144,10 +175,10 @@ class _MossCodecStreamSession: audio = audio_tensor.detach().to("cpu", torch.float32) lengths = lengths_tensor.detach().to("cpu") if lengths_tensor is not None else None out: dict[int, torch.Tensor] = {} - for slot in slot_codes: - wav = audio[slot] + for row, slot in enumerate(slots): + wav = audio[row] if lengths is not None: - wav = wav[..., : int(lengths[slot].item())] + wav = wav[..., : int(lengths[row].item())] out[slot] = wav.contiguous() return out @@ -201,17 +232,16 @@ class MossTTSCodecDecoder(nn.Module): self._n_channels: int = 1 self._sr_tensor = torch.tensor(self._OUTPUT_SAMPLE_RATE, dtype=torch.int32) self._stream_session: _MossCodecStreamSession | None = None - self._stream_slots: int = self._connector_int("codec_stream_slots", default=0) + scheduler_cfg = getattr(self.vllm_config, "scheduler_config", None) + self._stream_state_capacity = max(1, int(getattr(scheduler_cfg, "max_num_seqs", 1) or 1)) + self._initial_stream_chunk_frames: int = self._connector_int("initial_codec_chunk_frames", default=0) self._stream_chunk_frames: int = self._connector_int("codec_chunk_frames", default=0) self._stream_max_step_frames: int = self._stream_chunk_frames or 100 self._stream_req_slots: dict[str, int] = {} - self._stream_pending_codes: dict[str, list[torch.Tensor]] = {} - self._stream_starved_reqs: set[str] = set() - self._codec_streaming: bool = self._connector_bool("codec_streaming", default=False) - default_graph_capture_max = self._stream_chunk_frames or min(self._stream_max_step_frames, 32) - default_graph_capture_sizes = list(range(1, max(1, default_graph_capture_max) + 1)) - self._streaming_cudagraph_capture_sizes = self._streaming_cudagraph_capture_sizes_from_compilation_config( - default_graph_capture_sizes + self._async_chunk = bool(getattr(self.vllm_config.model_config, "async_chunk", False)) + self._streaming_graph_batch_sizes = self._streaming_graph_batch_sizes_from_compilation_config() + self._streaming_graph_frame_sizes = sorted( + {frames for frames in (self._initial_stream_chunk_frames, self._stream_chunk_frames) if frames > 0} ) # ------------------------------------------------------------------ @@ -275,7 +305,7 @@ class MossTTSCodecDecoder(nn.Module): }, ) - if self._codec_streaming and runtime_additional_information is None: + if self._async_chunk and runtime_additional_information is None: return OmniOutput( text_hidden_states=None, multimodal_outputs={ @@ -290,8 +320,6 @@ class MossTTSCodecDecoder(nn.Module): streaming_work: list[tuple[int, str, torch.Tensor, bool]] = [] if input_ids is None or input_ids.numel() == 0: - for i, wav in self._finish_empty_streaming_requests(info_list).items(): - audios[i] = wav.reshape(-1) if wav.ndim == 1 or int(wav.shape[0]) == 1 else wav return OmniOutput( text_hidden_states=None, multimodal_outputs={"model_outputs": audios, "sr": srs}, @@ -341,12 +369,7 @@ class MossTTSCodecDecoder(nn.Module): continue meta = (info.get("meta", {}) if isinstance(info, dict) else {}) or {} finished = bool(meta.get("stream_finished", meta.get("finished", False))) - streaming_enabled = bool(meta.get("codec_streaming", self._codec_streaming)) - code_flat_numel = meta.get("code_flat_numel") - if streaming_enabled and finished and code_flat_numel is not None and int(code_flat_numel) == 0: - for _, wav in self._finish_empty_streaming_requests([info]).items(): - audios[i] = wav.reshape(-1) if wav.ndim == 1 or int(wav.shape[0]) == 1 else wav - continue + streaming_enabled = self._async_chunk if seg.numel() % self._n_vq != 0: logger.warning( "MossTTS codec input length %d not divisible by n_vq %d; skipping.", @@ -414,40 +437,6 @@ class MossTTSCodecDecoder(nn.Module): multimodal_outputs={"model_outputs": audios, "sr": srs}, ) - def _finish_empty_streaming_requests(self, info_list: list[dict[str, Any]]) -> dict[int, torch.Tensor]: - """Release codec stream state for empty finish sentinels. - - Stage-0 can finish on a step that emits no new audio frame. The stage - input processor forwards that as an empty payload with finished=true. - If a request never acquired a stream slot, do not offline-decode its - buffered codes here: streaming requests must only emit client deltas. - """ - session = self._stream_session - outputs: dict[int, torch.Tensor] = {} - if session is None: - return outputs - for i, info in enumerate(info_list): - if not isinstance(info, dict): - continue - meta = (info.get("meta", {}) or {}) if isinstance(info.get("meta", {}), dict) else {} - if not bool(meta.get("codec_streaming", self._codec_streaming)): - continue - finished = bool(meta.get("stream_finished", meta.get("finished", False))) - if not finished: - continue - req_key = self._runtime_request_key(info, meta, i) - slot = self._stream_req_slots.get(req_key) - pending = req_key in self._stream_pending_codes - if slot is not None or pending: - if pending and slot is None: - logger.warning( - "MOSS codec stream request %s finished before a codec stream slot became available; " - "dropping buffered codes instead of offline-decoding a non-delta waveform.", - req_key, - ) - self._finish_stream_request(req_key, session, slot) - return outputs - @staticmethod def _normalize_seq_token_counts(value: Any) -> list[int] | None: if value is None: @@ -486,15 +475,12 @@ class MossTTSCodecDecoder(nn.Module): return None if self._stream_session is not None: return self._stream_session - slots = self._stream_slots - if slots <= 0: - scheduler_cfg = getattr(self.vllm_config, "scheduler_config", None) - slots = int(getattr(scheduler_cfg, "max_num_seqs", 1) or 1) self._stream_session = _MossCodecStreamSession( self._codec, - stream_slots=max(1, slots), + state_capacity=self._stream_state_capacity, n_vq=self._n_vq, - cudagraph_capture_sizes=self._streaming_cudagraph_capture_sizes, + graph_batch_sizes=self._streaming_graph_batch_sizes, + graph_frame_sizes=self._streaming_graph_frame_sizes, ) return self._stream_session @@ -511,40 +497,23 @@ class MossTTSCodecDecoder(nn.Module): max_step_frames = max(1, int(self._stream_max_step_frames)) for output_index, request_id, codes_nq_t, finished in items: - pending = self._stream_pending_codes.get(request_id) slot = self._stream_req_slots.get(request_id) if slot is None: slot = session.acquire() if slot is None: - self._append_stream_pending(request_id, codes_nq_t) - if request_id not in self._stream_starved_reqs: - logger.warning( - "MOSS codec streaming slots exhausted; buffering %s until a stream slot is available.", - request_id, - ) - self._stream_starved_reqs.add(request_id) - if finished: - logger.warning( - "MOSS codec stream request %s finished before a codec stream slot became available; " - "dropping buffered codes instead of offline-decoding a non-delta waveform.", - request_id, - ) - self._finish_stream_request(request_id, session, None) - continue + raise RuntimeError( + "MOSS codec state capacity exhausted. Stage-1 admission must count idle live streams " + f"against max_num_seqs={self._stream_state_capacity}; refusing to drop audio codes." + ) self._stream_req_slots[request_id] = slot - if pending: - self._append_stream_pending(request_id, codes_nq_t) - replay_codes = self._pop_stream_pending(request_id) - wav = self._decode_stream_slot_sequence(session, slot, replay_codes) - if wav is not None: - outputs[output_index] = wav - if finished: - self._finish_stream_request(request_id, session, slot) - continue - if int(codes_nq_t.shape[1]) > max_step_frames: - wav = self._decode_stream_slot_sequence(session, slot, codes_nq_t) + wav = self._decode_stream_slot_sequence( + session, + slot, + codes_nq_t, + terminal=finished, + ) if wav is not None: outputs[output_index] = wav if finished: @@ -557,7 +526,8 @@ class MossTTSCodecDecoder(nn.Module): for group in grouped.values(): plan = {slot: codes_nq_t for _, _, slot, codes_nq_t, _ in group} - decoded = session.step(plan) + terminal_slots = {slot for _, _, slot, _, finished in group if finished} + decoded = session.step(plan, terminal_slots=terminal_slots) for output_index, request_id, slot, _, finished in group: wav = decoded.get(slot) if wav is not None: @@ -567,22 +537,13 @@ class MossTTSCodecDecoder(nn.Module): return outputs - def _append_stream_pending(self, request_id: str, codes_nq_t: torch.Tensor) -> None: - self._stream_pending_codes.setdefault(request_id, []).append( - codes_nq_t.detach().to("cpu", torch.long).contiguous() - ) - - def _pop_stream_pending(self, request_id: str) -> torch.Tensor: - pending = self._stream_pending_codes.pop(request_id, []) - if not pending: - return torch.empty((self._n_vq, 0), dtype=torch.long) - return torch.cat(pending, dim=1).contiguous() - def _decode_stream_slot_sequence( self, session: _MossCodecStreamSession, slot: int, codes_nq_t: torch.Tensor, + *, + terminal: bool = False, ) -> torch.Tensor | None: if codes_nq_t.numel() == 0: return None @@ -590,7 +551,11 @@ class MossTTSCodecDecoder(nn.Module): parts: list[torch.Tensor] = [] for start in range(0, int(codes_nq_t.shape[1]), max_step_frames): chunk = codes_nq_t[:, start : start + max_step_frames] - decoded = session.step({slot: chunk}) + is_terminal_chunk = terminal and start + max_step_frames >= int(codes_nq_t.shape[1]) + decoded = session.step( + {slot: chunk}, + terminal_slots={slot} if is_terminal_chunk else None, + ) wav = decoded.get(slot) if wav is not None: parts.append(wav) @@ -607,8 +572,6 @@ class MossTTSCodecDecoder(nn.Module): if slot is not None: session.release(slot) self._stream_req_slots.pop(request_id, None) - self._stream_pending_codes.pop(request_id, None) - self._stream_starved_reqs.discard(request_id) def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None: """Release codec streaming slots when requests finish outside payload flow. @@ -623,17 +586,12 @@ class MossTTSCodecDecoder(nn.Module): for req_id in finished_req_ids: request_id = str(req_id) slot = self._stream_req_slots.get(request_id) - has_state = ( - slot is not None or request_id in self._stream_pending_codes or request_id in self._stream_starved_reqs - ) - if not has_state: + if slot is None: continue if session is not None: self._finish_stream_request(request_id, session, slot) else: self._stream_req_slots.pop(request_id, None) - self._stream_pending_codes.pop(request_id, None) - self._stream_starved_reqs.discard(request_id) def _connector_int(self, name: str, default: int = 0) -> int: model_cfg = getattr(self.vllm_config, "model_config", None) @@ -646,29 +604,14 @@ class MossTTSCodecDecoder(nn.Module): return int(extra_cfg[name]) return default - def _connector_bool(self, name: str, default: bool = False) -> bool: - model_cfg = getattr(self.vllm_config, "model_config", None) - connector_cfg = getattr(model_cfg, "stage_connector_config", None) - if isinstance(connector_cfg, dict): - extra_cfg: dict | None = connector_cfg.get("extra", connector_cfg) - else: - extra_cfg = getattr(connector_cfg, "extra", None) - if isinstance(extra_cfg, dict) and name in extra_cfg: - return bool(extra_cfg[name]) - return default - - def _streaming_cudagraph_capture_sizes_from_compilation_config( - self, - default: list[int], - ) -> list[int]: + def _streaming_graph_batch_sizes_from_compilation_config(self) -> list[int]: if getattr(self.vllm_config.model_config, "enforce_eager", True): return [] compilation_config = getattr(self.vllm_config, "compilation_config", None) capture_sizes = getattr(compilation_config, "cudagraph_capture_sizes", None) if capture_sizes: - max_step_frames = max(1, int(self._stream_max_step_frames)) - return sorted({int(size) for size in capture_sizes if 0 < int(size) <= max_step_frames}) - return list(default) + return sorted({int(size) for size in capture_sizes if 0 < int(size) <= self._stream_state_capacity}) + return [] # ------------------------------------------------------------------ # Weight loading @@ -770,6 +713,17 @@ class MossTTSCodecDecoder(nn.Module): device = self.vllm_config.device_config.device codec.to(device=device, dtype=torch.float32) codec.eval() + build_decode_lut = getattr(codec.quantizer, "build_decode_lut", None) + if callable(build_decode_lut): + lut_dtype = torch.bfloat16 if device.type == "cuda" else torch.float32 + build_decode_lut(self._n_vq, dtype=lut_dtype) + lut = codec.quantizer._decode_lut + logger.info( + "MOSS Audio Tokenizer LFQ decoded LUT: shape=%s dtype=%s size=%.1f MiB", + tuple(lut.shape), + lut.dtype, + lut.numel() * lut.element_size() / (1024**2), + ) self._codec = codec inferred_channels = 2 if "v2" in codec_path.lower() else 1 self._n_channels = int( @@ -790,7 +744,7 @@ class MossTTSCodecDecoder(nn.Module): ) self._configure_decoder_cudagraph(device) - if self._codec_streaming and self._streaming_cudagraph_capture_sizes: + if self._async_chunk and self._streaming_graph_batch_sizes and self._streaming_graph_frame_sizes: self._ensure_stream_session() # vLLM's track_weights_loading() compares the returned set against @@ -818,18 +772,19 @@ class MossTTSCodecDecoder(nn.Module): """Select the codec CUDA Graph path. ``enforce_eager`` is the single graph on/off switch. If graphing is - enabled, ``codec_streaming`` decides whether decode uses the persistent + enabled, ``async_chunk`` decides whether decode uses the persistent streaming-state wrapper or the offline full-chunk wrapper. """ if getattr(self.vllm_config.model_config, "enforce_eager", True): - self._streaming_cudagraph_capture_sizes = [] + self._streaming_graph_batch_sizes = [] return if self._codec is None: return - if self._codec_streaming: + if self._async_chunk: logger.info( - "MOSS-TTS codec CUDA Graph selected streaming wrapper: capture_sizes=%s", - self._streaming_cudagraph_capture_sizes, + "MOSS-TTS codec CUDA Graph selected streaming wrapper: B=%s exact_T=%s", + self._streaming_graph_batch_sizes, + self._streaming_graph_frame_sizes, ) return diff --git a/vllm_omni/model_executor/models/moss_tts/pipeline.py b/vllm_omni/model_executor/models/moss_tts/pipeline.py index 254bc18fe..f499654a4 100644 --- a/vllm_omni/model_executor/models/moss_tts/pipeline.py +++ b/vllm_omni/model_executor/models/moss_tts/pipeline.py @@ -46,6 +46,7 @@ MOSS_TTS_PIPELINE = PipelineConfig( final_output_type="audio", engine_output_type="audio", model_arch="MossTTSCodecDecoder", + retains_state_across_chunks=True, sync_process_input_func=f"{_PROC}.talker2codec", sampling_constraints={"detokenize": True}, ), @@ -76,6 +77,7 @@ MOSS_TTS_REALTIME_PIPELINE = PipelineConfig( final_output_type="audio", engine_output_type="audio", model_arch="MossTTSCodecDecoder", + retains_state_across_chunks=True, sync_process_input_func=f"{_PROC}.talker2codec", sampling_constraints={"detokenize": True}, ), @@ -109,6 +111,7 @@ MOSS_TTS_LOCAL_PIPELINE = PipelineConfig( final_output_type="audio", engine_output_type="audio", model_arch="MossTTSCodecDecoder", + retains_state_across_chunks=True, sync_process_input_func=f"{_PROC}.talker2codec", sampling_constraints={"detokenize": True}, ), diff --git a/vllm_omni/model_executor/stage_input_processors/moss_tts.py b/vllm_omni/model_executor/stage_input_processors/moss_tts.py index 2a7da4a4d..3ba32271e 100644 --- a/vllm_omni/model_executor/stage_input_processors/moss_tts.py +++ b/vllm_omni/model_executor/stage_input_processors/moss_tts.py @@ -294,7 +294,6 @@ def talker2codec_raw_async_chunk( meta=MetaStruct( req_id=[req_id], left_context_size=0, - codec_streaming=True, codec_chunk_frames=0, codec_left_context_frames=0, stream_finished=torch.tensor(True, dtype=torch.bool), @@ -326,7 +325,6 @@ def talker2codec_raw_async_chunk( meta=MetaStruct( req_id=[req_id], left_context_size=0, - codec_streaming=True, codec_chunk_frames=int(chunk_codes.shape[0]), codec_left_context_frames=0, code_flat_numel=int(codec_flat.numel()), From c2b55de5b9ea618cd296665c31c1ffb82b366900 Mon Sep 17 00:00:00 2001 From: Zhou Taichang Date: Tue, 28 Jul 2026 23:25:57 +0800 Subject: [PATCH 08/37] [Test] Give the logprobs scheduler doubles the vLLM 0.26 contracts (#5483) Signed-off-by: Gao Han Co-authored-by: tzhouam Co-authored-by: Claude Fable 5 Co-authored-by: Gao Han --- tests/core/sched/test_omni_ar_scheduler_logprobs.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/core/sched/test_omni_ar_scheduler_logprobs.py b/tests/core/sched/test_omni_ar_scheduler_logprobs.py index 130ac1c88..394063843 100644 --- a/tests/core/sched/test_omni_ar_scheduler_logprobs.py +++ b/tests/core/sched/test_omni_ar_scheduler_logprobs.py @@ -76,6 +76,7 @@ class _Request: self.status = RequestStatus.RUNNING self.sampling_params = SimpleNamespace(num_logprobs=1) self.num_computed_tokens = 0 + self.num_in_flight_tokens = 0 self.num_output_placeholders = 0 self.has_encoder_inputs = False self.pooling_params = None @@ -104,7 +105,6 @@ class _RequestQueue(list): self.remove(request) -@pytest.mark.skip(reason="#issue 5484") def test_invalid_logprobs_finish_only_the_affected_scheduler_request() -> None: """A bad runner response must not bring down unrelated batch requests.""" bad = _Request("bad") @@ -138,7 +138,8 @@ def test_invalid_logprobs_finish_only_the_affected_scheduler_request() -> None: scheduler.requests.pop(request.request_id, None) scheduler.finished_req_ids.add(request.request_id) scheduler.finished_req_ids_dict[request.client_index].add(request.request_id) - return None + # vLLM 0.26 contract: (kv_xfer_params, ec_xfer_params) + return None, None scheduler._update_request_with_output = update_request scheduler._process_kv_transfer_trigger = lambda _request, _tokens: False From c96fe54bd887a07c1ada6574b64a190d16f8c0ba Mon Sep 17 00:00:00 2001 From: Inesh Reddy Chappidi Date: Tue, 28 Jul 2026 12:27:51 -0400 Subject: [PATCH 09/37] [Test] Add regression test for MiniCPM-o 4.5 audio-stream output kinds (#5356) Signed-off-by: Inesh Reddy --- ...ing_chat_minicpmo45_stream_output_kinds.py | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 tests/entrypoints/openai_api/test_serving_chat_minicpmo45_stream_output_kinds.py diff --git a/tests/entrypoints/openai_api/test_serving_chat_minicpmo45_stream_output_kinds.py b/tests/entrypoints/openai_api/test_serving_chat_minicpmo45_stream_output_kinds.py new file mode 100644 index 000000000..1a3a4f22e --- /dev/null +++ b/tests/entrypoints/openai_api/test_serving_chat_minicpmo45_stream_output_kinds.py @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Regression tests for MiniCPM-o 4.5 audio-streaming output-kind coercion. + +Chat ``stream=true`` coerces every AR stage to DELTA. For MiniCPM-o 4.5 that +hands ``llm2tts`` one thinker token per step while the forwarded hidden states +still span prompt + generated, so the talker is conditioned on a broadcast +mismatch (issue #5298). The thinker must stay FINAL_ONLY; only the talker +streams. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +from vllm.sampling_params import RequestOutputKind, SamplingParams + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +MINICPMO45_ARCH = "MiniCPMO45OmniForConditionalGeneration" + + +@pytest.fixture +def serving_chat(): + from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat + + return object.__new__(OmniOpenAIServingChat) + + +def _stage(model_arch: str, model_stage: str) -> SimpleNamespace: + # SimpleNamespace, not MagicMock: _stage_get prefers .get() when present, + # and a MagicMock satisfies hasattr(obj, "get") and returns a mock. + return SimpleNamespace(engine_args=SimpleNamespace(model_arch=model_arch, model_stage=model_stage)) + + +def _minicpmo45_stages(): + return [_stage(MINICPMO45_ARCH, "llm"), _stage(MINICPMO45_ARCH, "tts")] + + +def _delta_params(n: int = 2): + return [SamplingParams(output_kind=RequestOutputKind.DELTA) for _ in range(n)] + + +def test_thinker_forced_final_only_talker_stays_delta(serving_chat): + serving_chat.engine_client = SimpleNamespace(stage_configs=_minicpmo45_stages()) + + out = serving_chat._fix_minicpmo45_audio_stream_output_kinds(_delta_params(), ["text", "audio"]) + + assert out[0].output_kind is RequestOutputKind.FINAL_ONLY + assert out[1].output_kind is RequestOutputKind.DELTA + + +@pytest.mark.parametrize("modalities", [None, [], ["text"]]) +def test_non_audio_request_is_untouched(serving_chat, modalities): + serving_chat.engine_client = SimpleNamespace(stage_configs=_minicpmo45_stages()) + + out = serving_chat._fix_minicpmo45_audio_stream_output_kinds(_delta_params(), modalities) + + assert [p.output_kind for p in out] == [RequestOutputKind.DELTA] * 2 + + +def test_other_architectures_are_untouched(serving_chat): + stages = [_stage("SomeOtherOmniForConditionalGeneration", "llm")] + serving_chat.engine_client = SimpleNamespace(stage_configs=stages) + + out = serving_chat._fix_minicpmo45_audio_stream_output_kinds(_delta_params(1), ["text", "audio"]) + + assert out[0].output_kind is RequestOutputKind.DELTA + + +def test_shorter_sampling_params_list_does_not_raise(serving_chat): + serving_chat.engine_client = SimpleNamespace(stage_configs=_minicpmo45_stages()) + + out = serving_chat._fix_minicpmo45_audio_stream_output_kinds(_delta_params(1), ["text", "audio"]) + + assert len(out) == 1 + assert out[0].output_kind is RequestOutputKind.FINAL_ONLY From f6e59c329b01c43750a425893cde55b0aacf25cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E8=8A=B7=E7=91=9C=EF=BC=88Zhang=20Zhiyu=EF=BC=89?= <129153579+Zhiyu0603@users.noreply.github.com> Date: Wed, 29 Jul 2026 00:38:37 +0800 Subject: [PATCH 10/37] [Bugfix] Return diffusion metrics from /v1/images/generations (#5278) Signed-off-by: Zhiyu0603 Co-authored-by: Zhiyu0603 --- vllm_omni/entrypoints/openai/api_server.py | 19 ++++++++++++++++--- .../entrypoints/openai/protocol/images.py | 4 ++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index 5338761ad..44225b69b 100755 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -1830,10 +1830,17 @@ async def generate_images( status_code=generation_result.error.code if generation_result.error else 400, content=generation_result.model_dump(), ) - flat_images, _, _, _ = generation_result + flat_images, stage_durations, peak_memory_mb, _ = generation_result image_data = [ImageData(b64_json=encode_image_base64(img), revised_prompt=None) for img in flat_images] - return ImageGenerationResponse(created=int(time.time()), data=image_data) + return ImageGenerationResponse( + created=int(time.time()), + data=image_data, + metrics={ + "stage_durations": stage_durations or None, + "peak_memory_mb": float(peak_memory_mb) if peak_memory_mb else None, + }, + ) # Build params - pass through user values directly prompt: OmniTextPrompt = {"prompt": request.prompt, "modalities": ["image"]} @@ -1899,7 +1906,7 @@ async def generate_images( logger.debug(f"Generating {request.n} image(s) {size_str}") - # Generate images using AsyncOmni (multi-stage mode) + # Generate images using AsyncOmni. result = await _generate_with_async_omni( engine_client=engine_client, gen_params=gen_params, @@ -1929,10 +1936,16 @@ async def generate_images( for img in images ] + stage_durations = getattr(result, "stage_durations", None) + peak_memory_mb = getattr(result, "peak_memory_mb", None) response_kwargs = { "created": int(time.time()), "data": image_data, "output_format": output_format, + "metrics": { + "stage_durations": stage_durations or None, + "peak_memory_mb": float(peak_memory_mb) if peak_memory_mb else None, + }, } if request.size: response_kwargs["size"] = size_str diff --git a/vllm_omni/entrypoints/openai/protocol/images.py b/vllm_omni/entrypoints/openai/protocol/images.py index b7247962c..e83c7b5c0 100644 --- a/vllm_omni/entrypoints/openai/protocol/images.py +++ b/vllm_omni/entrypoints/openai/protocol/images.py @@ -190,6 +190,10 @@ class ImageGenerationResponse(BaseModel): description="Chain-of-thought text output from the AR stage. " "Only present for image editing (IT2I) with CoT-enabled models.", ) + metrics: dict[str, Any] | None = Field( + default=None, + description="Per-request generation metrics.", + ) def stream_response(self) -> StreamingResponse: if not self.data or not self.data[0].b64_json: From 0feb92bdfa7aada993d79f89f26c999c81cbe937 Mon Sep 17 00:00:00 2001 From: Sy03 <1370724210@qq.com> Date: Wed, 29 Jul 2026 01:58:34 +0800 Subject: [PATCH 11/37] [Bugfix][Minicpm] Support full-duplex three-stage serving (#5380) Signed-off-by: Sy03 <1370724210@qq.com> Co-authored-by: Hongsheng Liu --- .buildkite/cuda/test-merge.yml | 25 ++ .buildkite/cuda/test-nightly.yml | 8 +- .buildkite/cuda/test-ready.yml | 27 ++ .../app/static/playback_worklet.js | 15 + .../minicpmo_4_5/response_required_16k.wav | Bin 0 -> 175058 bytes .../minicpmo_4_5/soft_interrupt_16k.wav | Bin 0 -> 820780 bytes .../sched/test_omni_ar_scheduler_streaming.py | 43 +++ ...mni_generation_scheduler_update_session.py | 27 ++ .../test_invalid_minicpmo_4_5_omni.py | 5 +- .../test_chunk_transfer_adapter.py | 63 +++- .../offline_inference/test_minicpmo_4_5.py | 2 +- .../minicpmo_realtime_duplex_scenarios.py | 21 +- ..._minicpmo_realtime_duplex_multi_session.py | 102 +++-- ...minicpmo_realtime_duplex_soft_interrupt.py | 53 ++- tests/e2e/online_serving/test_minicpmo_4_5.py | 18 +- .../test_minicpmo_4_5_duplex.py | 147 ++++++++ .../test_minicpmo_4_5_duplex_expansion.py | 86 +++++ .../test_minicpmo_4_5_expansion.py | 20 +- .../test_minicpmo_realtime_duplex_drivers.py} | 65 ++-- tests/engine/test_multimodal_accumulation.py | 3 + .../test_orchestrator_stage_input_bridge.py | 71 +++- tests/engine/test_output_processor.py | 123 +++++- tests/engine/test_single_stage_mode.py | 54 +++ .../openai_api/test_duplex_handler.py | 335 ++++++++++++++++- tests/entrypoints/test_serve.py | 7 +- .../test_minicpmo_realtime_web_static.py | 65 ++++ tests/fullduplex/minicpmo45/test_input.py | 48 +++ tests/helpers/minicpmo_4_5_duplex.py | 179 +++++++++ .../minicpmo_4_5/test_code2wav_batching.py | 194 +++++++++- .../minicpmo_4_5/test_talker_batching.py | 180 ++++++++- .../test_minicpmo_4_5_async_chunk.py | 249 +++++++++++- .../test_minicpmo_4_5_omni.py | 114 +++++- .../test_qwen3_omni_streaming_helpers.py | 42 +++ tests/test_config_factory.py | 2 + tests/worker/test_native_duplex_hooks.py | 285 +++++++++++++- vllm_omni/core/sched/omni_ar_scheduler.py | 14 +- .../core/sched/omni_generation_scheduler.py | 15 + vllm_omni/core/sched/omni_scheduler_mixin.py | 5 + vllm_omni/deploy/minicpmo_4_5_duplex.yaml | 3 +- .../distributed/omni_connectors/adapter.py | 13 +- .../chunk_transfer_adapter.py | 17 +- vllm_omni/engine/async_omni_engine.py | 10 +- vllm_omni/engine/orchestrator.py | 27 +- vllm_omni/engine/stage_engine_startup.py | 244 ++++++++---- vllm_omni/entrypoints/cli/serve.py | 2 +- .../fullduplex/minicpmo45/data_plane.py | 11 +- .../fullduplex/minicpmo45/input.py | 104 ++++-- .../fullduplex/minicpmo45/stage0.py | 63 ++-- .../fullduplex/openai/session_runner.py | 42 ++- .../models/minicpmo_4_5/batched_token2wav.py | 14 +- .../minicpmo_4_5/minicpmo_4_5_code2wav.py | 353 ++++++++++++------ .../models/minicpmo_4_5/minicpmo_4_5_omni.py | 58 +-- .../minicpmo_4_5/minicpmo_4_5_omni_tts.py | 131 +++++-- .../minicpmo_4_5_omni.py | 257 +++++++------ .../stage_input_processors/qwen3_omni.py | 52 +-- vllm_omni/outputs/multimodal_accumulation.py | 1 + vllm_omni/outputs/output_processor.py | 20 +- 57 files changed, 3518 insertions(+), 616 deletions(-) create mode 100644 tests/assets/minicpmo_4_5/response_required_16k.wav create mode 100644 tests/assets/minicpmo_4_5/soft_interrupt_16k.wav create mode 100644 tests/e2e/online_serving/test_minicpmo_4_5_duplex.py create mode 100644 tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py rename tests/{examples/test_minicpmo_realtime_web.py => e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py} (96%) create mode 100644 tests/helpers/minicpmo_4_5_duplex.py diff --git a/.buildkite/cuda/test-merge.yml b/.buildkite/cuda/test-merge.yml index 96ea162b4..d72b8f287 100644 --- a/.buildkite/cuda/test-merge.yml +++ b/.buildkite/cuda/test-merge.yml @@ -169,6 +169,31 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate + - label: "Omni · MiniCPM-o 4.5 Duplex Test" + source_file_dependencies: + - tests/assets/minicpmo_4_5/response_required_16k.wav + - tests/assets/minicpmo_4_5/soft_interrupt_16k.wav + - tests/e2e/online_serving/test_minicpmo_4_5_duplex.py + - tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py + - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py + - tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py + - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py + - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py + - tests/helpers/minicpmo_4_5_duplex.py + - vllm_omni/deploy/minicpmo_4_5_duplex.yaml + - vllm_omni/experimental/fullduplex/ + - vllm_omni/model_executor/models/minicpmo_4_5/ + - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py + - vllm_omni/core/sched/ + - vllm_omni/outputs/multimodal_accumulation.py + - vllm_omni/engine/stage_init_utils.py + - vllm_omni/engine/async_omni_engine.py + - vllm_omni/entrypoints/cli/serve.py + timeout_in_minutes: 50 + commands: + - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'advanced_model and cuda' --run-level 'advanced_model' + mirror_hardwares: h100_2 + - label: "TTS · VoxCPM2 Test" source_file_dependencies: - tests/e2e/offline_inference/test_voxcpm2_tts.py diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index a82a4aab5..72b31d0ce 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -15,7 +15,13 @@ steps: - label: ":full_moon: Omni · Function Test with H100" timeout_in_minutes: 120 commands: - - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py + - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py + mirror_hardwares: h100_2 + + - label: ":full_moon: Omni · MiniCPM-o 4.5 Duplex Test" + timeout_in_minutes: 50 + commands: + - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni" --run-level "full_model" mirror_hardwares: h100_2 - label: ":full_moon: Omni · Doc Test with L4" diff --git a/.buildkite/cuda/test-ready.yml b/.buildkite/cuda/test-ready.yml index 664c6c711..b0ea85900 100644 --- a/.buildkite/cuda/test-ready.yml +++ b/.buildkite/cuda/test-ready.yml @@ -210,6 +210,33 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate + - label: "Omni · MiniCPM-o 4.5 Duplex Test" + source_file_dependencies: + - tests/assets/minicpmo_4_5/response_required_16k.wav + - tests/assets/minicpmo_4_5/soft_interrupt_16k.wav + - tests/e2e/online_serving/test_minicpmo_4_5_duplex.py + - tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py + - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py + - tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py + - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py + - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py + - tests/helpers/minicpmo_4_5_duplex.py + - vllm_omni/deploy/minicpmo_4_5_duplex.yaml + - vllm_omni/experimental/fullduplex/ + - vllm_omni/model_executor/models/minicpmo_4_5/ + - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py + - vllm_omni/core/sched/ + - vllm_omni/outputs/multimodal_accumulation.py + - vllm_omni/engine/stage_init_utils.py + - vllm_omni/engine/async_omni_engine.py + - vllm_omni/entrypoints/cli/serve.py + commands: + - | + timeout 20m bash -c " + pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and cuda' --run-level 'core_model' + " + mirror_hardwares: h100_2 + - label: "TTS · VoxCPM2 Test" source_file_dependencies: - tests/e2e/online_serving/test_voxcpm2_tts.py diff --git a/examples/online_serving/minicpmo/realtime_web/app/static/playback_worklet.js b/examples/online_serving/minicpmo/realtime_web/app/static/playback_worklet.js index c3216fd7f..f0d3d42ae 100644 --- a/examples/online_serving/minicpmo/realtime_web/app/static/playback_worklet.js +++ b/examples/online_serving/minicpmo/realtime_web/app/static/playback_worklet.js @@ -6,6 +6,7 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor { this.playedFrames = 0; this.underrunFrames = 0; this.drain = null; + this.terminalFadeFrames = 0; this.started = false; this.activeResponseId = null; this.initialBufferFrames = Math.round(sampleRate * 0.2); @@ -31,6 +32,7 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor { } } else if (message.type === 'drain') { this.drain = { responseId: message.responseId || null }; + this.terminalFadeFrames = Math.min(this.fadeFrames, this.bufferedFrames()); if (!this.started && this.bufferedFrames() > 0) { this.bufferWaitFrames = 0; this.startPlayback(); @@ -42,6 +44,7 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor { this.playedFrames = 0; this.underrunFrames = 0; this.drain = null; + this.terminalFadeFrames = 0; this.started = false; this.activeResponseId = null; this.bufferWaitFrames = this.initialBufferFrames; @@ -76,6 +79,7 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor { this.playedFrames = 0; this.underrunFrames = 0; this.drain = null; + this.terminalFadeFrames = 0; this.started = false; this.activeResponseId = null; this.bufferWaitFrames = this.initialBufferFrames; @@ -116,6 +120,7 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor { while (target < output.length && this.queue.length > 0) { const pcm = this.queue[0]; const count = Math.min(output.length - target, pcm.length - this.offset); + const remainingBeforeChunk = this.bufferedFrames(); for (let index = 0; index < count; index += 1) { let sample = pcm[this.offset + index] / 32768; if (this.fadeInFrames > 0) { @@ -123,6 +128,16 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor { sample *= elapsed / this.fadeFrames; this.fadeInFrames -= 1; } + const remainingFrames = remainingBeforeChunk - index; + if ( + this.drain + && this.terminalFadeFrames > 0 + && remainingFrames <= this.terminalFadeFrames + ) { + sample *= this.terminalFadeFrames === 1 + ? 0 + : (remainingFrames - 1) / (this.terminalFadeFrames - 1); + } output[target + index] = sample; } target += count; diff --git a/tests/assets/minicpmo_4_5/response_required_16k.wav b/tests/assets/minicpmo_4_5/response_required_16k.wav new file mode 100644 index 0000000000000000000000000000000000000000..ae405249343c7081fff7929ddde44d0dfac56890 GIT binary patch literal 175058 zcmX6_1$Y$65*}MiHpwQt8&@J|utN{G;C{#fhr9dT0mpIp;qHgKySqEVH9#N<@lDpp z-{1Sb`DQXR-P2uN^;dOuRd2eqPD^`uh@pBk>E2@Kn91=bilS&7vl`;&FN$I)6E%3i z+b|M1|ts zNU;>JYA7pJqUwETsak|1l+sW+Xt6;3f-hgF`RYp?k8qq+cdP%YN7Q@jE%mN?UA?28 z#d%oWkCG$mRrR!bS-qy7S6}0sbR6&T^#|3fs%ozK9e1D9JhWPjwqJqr07^>C4WWvCG-;`P)PAbk)vx&bSbeTOLn~j<_G9(4`c}Wv0R~?gT2Filt(3n(!P3ItWjqafVQpz_U=Q z2GxpcLUp3jaIHqQpqk;iGmidL8(dpceW;F94_y0FBdBiFAlwb0hEZdwMpQ4n`32`7 zY9vY~;dwMQgBn9krlwF+eYGZ1KhQ@0#|;h9OYC6)eN4Lr|S96HdIygoS|6dmqsF zE2wFVzk{eQRC7>EXimaiOR6T-l!~KjQw>3REqosX&N|ds$_^8AVS> zurcncQT4$AJ?dM~nhSJ0!OiQ;5Sx(*ahQ75Y7K>H;1 zPxW_oh&mH@bJgYQ9Cb6su}s~Gzw0oneW<%kU8%0ZTk>w6xt(gQt~1ogO15_>BaWfzajvXpbMtXkfvux}eARA!VYqguACf z^%=B(2;5zz9#dDVi*UDGU9A2ADo3j0@pqisU+o8K7l6K1>ObmKydA8L#aIUU{{D*R z-|>Dlu1oRv4|Suu0d21ZKQ61s!JC(A3Ho-cfuO_;8B_wE8iAj+fR;39M{nrE7-|-^ zfLcWTM{UJ*F|~zSP3@r8Q%k75)HZ4bH5bogQEM(VV>Hzd8a9FIj<0J1^~t!#`=rgQ z4D}AU&4vwljd9w+r8mICS6tIE@<%}Wb&T^g@N@%Dmm#s6pzs#7;1aas8o2Qoc7@RV z0(1~zpT{LvHEbIeERYYAqY8b$A8T@ub zPOpLSkI*nxJpsz{p?{^gC$cj@kAt9VJbI$hx)<0L)CgE?iYkPD3b1YpN@aYPiSejD z+xr0+u%o09G?qgCgth{-#h@jAlfev8zHOyz4)nf7jUrG}2&_Lsf6wstDMs`jns*L-ehOM1piT}( zS^$c2aPI;oMzlxh3Bp(?^ji_Otrqm^7wBLuJT-bRNhK;tICes+hpjR4(ks1A77 z3;Ng*Z#%)ZbcT+0h13V&{#WQ~Ta-1#h^k|ZH1tscm$cwP38d%57#_gNy#w`6VIeMp z=3B6LCn5Enc;1cc1!&xA;NUPc?J_jrEXw|aHr&ShQ+U1w9o`D0>_SV##_Wd`I*<3~ zaDE3;i2mG#-O7hlGQkHsc*kL$GWcZg1ovYwY7@>fxauGW3vd$zT*Si`Rt8oof-j+X zN34kzRwWoxu|h7<(2`V0Dik%#qcjmV*o=}W)D6T*TJu0w3h?wDoc;{kPHZAbN z~(w628xbX1v9hpU~PZ+&=`CFXAZ^UwlDJx!^tzJo;gVeTVl&z;!XC@*Vx; z;;{KN!fu?C=a6&rqMB&TWi37o+@$ zcAw!s2MFZQOBN{1fF#Jw^hl>#0$xE2HLxxR8@ABcYCp^XpFMj6=l z;JFaK29Y6++Isj(#P3m2_Y+#m0%zW!-%QZ=6(hKh=ezjs9Vn-$H$Y(^dSPIJlYA15 z1OgHvJtL$a2yF|3{IvK>ppkcNIbK3GccEqH!HtK| zuv`r|>54$PST;8`HXra&Tb&^i-(PC(zpN=1PW|L0{Oc=Z|5ClXUopUBON z7F3)i7!`$5JGAi&#`g+R%7xZ^h2$PXJ{geSM_h>)>OkoiNVXU<%>e%ho`{^Cc=O4p zF-~yGtA?X*8uLmN#%_g;s|rg;Jd^s6O&yG`I?nny$h;SadXb>EHEPrce=6e|4x3Q{ zHOZWq2o9#AjRrWXVq8DreRG`kU^i1xyDHlC$EYj9%9vnn^x%a77r|wZJ+nq1icds#R4UOf!uMIef4{rfat(apPfQIt;E*R8RKwruD zCJ_2zM&Gfxqd-$K#v2VvjHpTOtDu$Y7;yvCuZz(oL8|4@b3=?G90(0S%}_|#4?T0} zrLvDVb%BB?96|Vs0)okU#sbQn7>5j5YNkI~92co5HE2Oh&l7D3ho?Vo}4udqtl zD18Tg{f57f@zs0u^vY*db5QFxXnF1GFIs}Lzu-v2oSXo=5Di(zfx9u_q6cHi1wy~#BDl#toe1_^wDy?3A<1`JdZtt`bGVpBi+bbyC0 z5MMd~<$l1Q4CHa(dUc>W2#7QR&n{>@nJ)sMMP%;K;a&?~24K}(fP+}{Ld-Evod4H* z9r#ReD*D#e#B&IP-Z5x74jNDsXH!VD5A1VK_(QF*(&~xT)p&SFL-BVAB-;-@QWyBo zO(C@~ob`c-^01Ea(4z9ti#TWld6NV^iv(U`(W45RNHp973&deGJgklmdg2fJ?vGMG z&_n!Cf4rkGJ__$Rj4}oX(dQ7WZ@w32`{vR_yA7a1w%qbxFhqWKPV!zj1yFR!1E)tkqQ21Lmp;~jjWB# zuqv_ORUEiP_!R_O6AP^(qa~~C1W2L`{$y`|vXFsfc2jQp>t!WI6YX~U}!EDqO zchw-3?(kUu@3*!?otpTjGc<2J-VH{Cpd+-iu}@2XhIGmSC28nagZ^2_hWK&$pt=BK z{esaGYxNv_y9qu$1&)`2ODnJ%`wQ6LgjME3^>3^ym#VX{5}SdyYjJK?{{{c{U_G`L z#~B}2H~Uu4PoRJAV5Q!|vRr~j+=NVS!t$JgUc3R99zafyeID)uNP=h~(U|+ttVbw0 z0(o43&Af~IJCF~F^_+lY9zZfTK<{nv-v)U|zyk{_?T<4UrykrSo-Pd@MgpItj98gy zU?>Iet+4oEz8aw@H~Bm-62T&#ZV+tIcTg-td*6fRJD}t!Xuap7>>?<<1Y2^#N84_U z;|MIlR&Zc5{+>nYDR}uO!GoKi`zt6Xv5#wL>n2(`iPD>>a~{{X;PV%thWL*qYBWHirFj-GiVH`*FbIrGhsMGAbBfllZZzJXy8v6j~*Bc0SAb0 zNBk%Q#vXun!f+DqnD8V3r6KSp0x(ml7~xmYMr_?hXu&p&VG$(06vq;*y=P;UvIi^l z?YQ5Hh(dR)w`bzoOYNX`!QXCbYqbgPdZ@i|-(UR&>-b?Z<~sCS1vCgrK;r&&y~IOn1g=tT$pKLojpn zz|$~lHtr`=(|q=+JN&c0uvb0crS*gDY5~lr0qeD3ui66x)xpDBz8R|wct&E7Uc~rJ z=qCkzm~m7_9kQ+`eohpuX*jrB7DqU~3WYrUaTktjB;JuVmJz2H>tH{i#S0rNK#z){ z>0}l20!Sk3R}xiJ)i8`M3H1UYTN?6;_RX5rp}9?Pwu4raDAov!^|R{#Co%QtCf6| z=D}8nV%{NIB4T{Ruaco9M(~CBWD>kd!ZETQAvhy`6AfEkgfAn3UZSBg<~a&8lmOI` zHUAgfeFUe9(P}2zCF_qb;NMHoa08SO4gU^{kpUXX3P#0WVqG;rFbk{Y2iynZh=Dl z_7pQm0a_%Hvk!Q028T30-sORl`Iv)=??!k|ynPbYAn~LR;K)0)mW>(i1Fj#?-cz*l z8TK*J9VJi7f-*aBh;bL z^TBu?3|;SzzrP`VHxXr{F(-^i9B(b=kJUbp_BeVVvDPE#i`esf=>Hb_E(Xmcj!KY3 ztW*Gy8jphoS}SAbs^as?>*HDuva5`vBIH*awUc1^h#tq{ITXl>gDfKvqlrcVMJ-~t5Uq(&N5X@!1CT(3Ups)4u7VeM-}A_=ha#Cs;5XE87@!bA4Ly5vFL6h$H(E+Fn3 zWKZIxSwQ`JXhbHSUO`jd;CK%`xrdqhGxX{fH0~M7Nk)mRuD@Yse}(d+D8B&hxr~|w z%jtMWbd9W=h=v+r^IXt;;`aq1TFAr7GQPRl227B8A5oJ5Hom~_*>JkC>VE^zlEh%i zN|N~bBw|5)cP&OlR$zYMBEei^NQ9uJGRB;Q`^LUC89`4ujK3B*O`n^obtB#z$wxf~RV5A?Ai z?z*DQHo$pJXhsmQDPh$>>`?+nOR}H_$e2h;LVnbYC$cX50nQNZN%47a(cqmC6tnoA ztQ3h=Gs4!9ydjCzy~6Ryw_ZI99+CLv6^!y2IG7Cte#4kaJm|U4Grt2KyvAHYG7*{J z;b-vSC9wD!lDP{Mzk_WgGm95^&4o0GMP_~I5m56RO26Q50jLN7qDp{te_)t+bw=x3Oq^uBl?@6=jIbEkDgpJxtIPp)A260&P(@b0EVx6k#X-(AkVZ1B zN%)%#4N8HgkZeH>SnN2T&5FfcCD@u8up%{~Lv>J#Xjyrb*N5JbEJ7;2Pr*Hjb5(&% zNvx;=o=N0209r>@>XE1s4vAXOC$a63puz^4zkwG{97I|-q3J|pUSP((1+6&@)NX=y zZ3m9mLchp48-DX@pm-JVz68kL3tTVvA$+4RJGLC&`aFEK5A7TPhjxHhm!NsqAB_O8T!)%y4KM*Z*|6+wi0xV#8=Z$zcze+vKnZE8f~GS-Jy9TqSOraYobk( zntYR42{t4|QL5_7HFMsq) zY_0`Y%4a*1A?uoWOJbkZK|^QEt^F{%L60CuMPsG1wNS+fpZbyz8hM?fXe?zK@4a`i+UuVY6hl=29jL4fYT3SBzXf8 zOAbTpv7jp+d>6n05@Gm1A3}h7vU7uI2(hXpYEles_dwpCp*;ma!YiM&iSECIR*{up zDQJoV4Ng!)VIHE(VQ+h85`FnE$9}YBhR{q*pVP3Q{ximSj+CsWYiNxzHPVtaLrQb}n;w^OW2bBPw~N=DPfQ=Clv%{Q z1@33Vv+Jc+pnA}Ess79r`Ux|PImupS`p_O_pwe9FKs{0fdMC33(FQ+uCiRDWUtL2F zrE4%O+kw(44dg(%zvL7L$W16OEip^jv1|gnj;+8XuxFS&W+z=p-=xFnC^=GUAjAsC z_&SnZYNc#dA5d8764fbGI+eyWWNxb_`GWLTelO0JS1TLT zqx7$A8rPYf#x$pvfU7N}3~`3kQ!0^OD@~MZ%6lb+nx!T)6}Vp)YmiRB5dZe5B-9Ppt{ors;?BF?iMDBmzd>rHKsXj z(A=diz(SUzMhN{qt%PMx#{Jgi*tA z9m+gqq#938XNPOOTno(=?Fvnrb|iO5yOCq`(fWP*5ZzGSF5L%hh^~r;*B;X)YEyOD znuD6*+AmA##&2PsRBXC2SGR@+HYC z-jTLTrzKSim(t{+@=GW&#hjg4F(*LX9UPG+kH@_0&7>S9_$?Ue5b3%ytQ@}BX0JN_?mhgeo>qEvys{Ve;dqow=8Wv}Ep?P}}H za-MU4;;#zz)Q3zw`+;r4ma=!KSbBo`MmDRHl`uL&Z9^}lda=da3f%(3F28hRuCZ!B zC&O-S2DhJ!Vw-F2y0?C7O&x-7gkBDHhg+g;5qqN-MZ|}{GbGYAopXx)^X6x*{ju}g zr-J)=6TEKeALgPyQ$Hh+_iygMB48i8f+^2S{1V5y(!cE)#c}pNg%_Po+$Y5lWv)Dp z9jvs{%wu}8?U_1KOLdvMoRH`~?|taq;5jMQln<&;mBrKssl2e$>+mLcv)wh^2iz?^ zf!@u0Yk7_mPCupRvk$m~ngiT7=4bkt`WtnI8b6Ri!STSmnu&Pe#JOnm60fb~?e^FV&Y=eUonUk>BsgBfxe=eNyUh!P_q4Slfx6XPdC zTf;q0t9i)Y)YkUb1l|n!%N!cMF~SvH7&o=dl;r-2ts@GIp;S%hh=Y#kZ;U_-8JJ8+Uebk-h&E|VaZbeoq zP!}-wETgAjwi!(6)zk78<(2YWvCw7d0rU`h25n?RHJ7#T488qM8lMMb2aNQ4s@=)V zVa70S&d*RKut)Iv(0Q_Lx9JA0=ObDmeYmvN`#1HTvM{1=|< z#(%>*C(lXx9RGLpnV^5!(fltCdx7UiQU1NW8n$Ick4x9u=a+VImJ!B_v2q%-io421 zYeICjwKv!?YN*mn=p$5hOWsEAAKt#aEWH);#moH9yxSwVDmc$LAG?emjrgb1oLRy= z(Vo%qTyu6NeL%g7nu0V&>Mp&JYtcv9-!!*%)%@Q1IsGOCEDszKnBxCbm!h4{j$=&R zaKolRk2xj$WAu~w2W7UUZlIj_KJqpxLwX_Glo&dKZKHjpOEV<+Ju?jVU*&&4;H2@CVTZ=0-J%WC-SoR_yb(|# zBt7g^tShOo!iCD^s#%jJht+3~InKXt^JLhqcQ;nutCe0q>x}asea*0^3B8iqB-D%= zWj%AtG_D%E zp6Z~=a)A6=E|epw)wG2@%l)C%8#RHV=}AbqX_+y`Z>?dk{<7b1{;t4h!6A{>gbQW+ zRh?DqZM6$YEdo_n_fMnm*e?W}yKr&ay^Ejfy7G*>Vy{)cR`Gp>CNU|NCHi;rF6T;H z%fgj~--|L!UYEo=TiZK2?%RH|TkID+Ez}hHs5U`6Sg+G>&|YBf%9JqL_0loasd%pO zv*iz%Ew51iO1xB_@9do9m}oPVbhG_lI>H$&q^tiiVYaqCmdP-}e1NRuNlE8x$7e%N!Kny--!uk;5E%XFPJMQWX zJ(OIrmr%ts-M+cBf62F!IL8EcGXGvWtM+5YanahXy1Ir!{dB#*&aRowwq`gg6FjIu z7jriZ4%0ewAFIVY+5E%Q#B?a|Q%G^hK&w9bW5S4vr>ixov8DXMh%wU5ALSohyma`K zd_MAK_t!5AhjBN;h9-_Fw?4UTOj+v@<8d}k87Ut3IP8(my7q8;K=IPz?Zx}^Qi}(c zSUpV@g?sOp=hxBiZ^K8wP26JYvv|-`&AG7TO6f8CUdK6idH$t1Sow@#BEJup-h2&aO^WN}VF-Mc0SL#;nYHGQ-GNzNxkPj{I+&Q)5?3hbG9*p}^>VD(j zFS}Q%2Bxa!ygoK?Nl22lpS8WYddQ>T8m4&D zgWv(yw8*r$mSs0pnN!uT92HSOU(28I%yqfb`P%1gH)p;cmc3L8_S+NsKKfq7`S8xc zJ&j+D-TW>73EIib3_07QDLGM?lhZu=Q&vfK#iD7&t@vE3y=H%azuz;zU-iAY5xN{i z6!u9I`KKP2lW}w^9awzQcGVu}-0oQ~#7P^J_Hui^JUDOGu*{=}Yx zT?)}H*SzLFX&!MsxMyr0cU#-S@Gxju@bHj5A$3B=TmB3wV|id1Zw(0l7*#bsFF7>z zTk5HleZk|nbpbE*6}`9k4A#}r|yc#g@Wt8jTtDBDRtTAQQ&$km|7DW`?T-u{lo z#lwm!+NPBhI3rz+;l;J(8EG`Xm5=d`LcW-iOp&HbrpFk`=g?1)uILHnHk8{^(NdN@i3Ep9-GcT zx{}VnHh)`Le8gUj&l5IMyVQBwS^-k<(eR;Rhb&`5h6QADZG~5!ZzX+7qwRO>JBnu8 z5}nH&UT-7EO1?4tM?v<|WAvRhak_Z6KT}h!B);)7?n$mhN22Sp{kHRjYoD{1tClOk zQ`uWZ=q)Z2Z_2&oeOR%rmco^OR0s9|m!`X-J+GJj-uU+ohzNWhqO+_BT^QQT+QxF& z^dY!w;9%2Y)0L3Nk=LTCC%#U8m-Hr*4`P^vqGn$oz8m~8=WWxBf3i;I-Yx#kw#WTX zso+Qv4l8-OP-Bt#Nk~9Qy`Z-K5t`3(MQM^ZN8Bav6b#->_ZGgJGuOGqUTS|_Jg9UN zUrrb&%@Us|v*j$Qp`3zso{Uvzd2xWUSh^*K3Lm{cgdLv0yyv}Vy)}g(Aw$gODQS^( zQi;R5sX1LtwO}T2!=NJz3|?ba&?0m5&?8~f!g4~lSktVVEw3!qES0Ulg*FOjB8%fz z#oUXyYCPn>N2zDOm_0Q+DBGSnIQK$sc)_sTAH~m#<`%pz8eTe>FHvS|_v<(7e;D%h zWpqDrA|ebsneW=xTurt!yPFxUPL|FK$J~>!t@DMWzq7yNtm~62%{{~2#JSSm+`h}X z$^D1t5?@j1D@(}3{Hz$|Wy)#gnmQUWuo2Wsx+AlkX#+1S1NYyl2zZfInagZH?ue$5 zKGVNdVAr5&L48bRf=$6EOoL4ugN6qEVoDEs6nHS`MbM+5xyC_$F5PKvAM>AdOknwi zQcJ9#()bYmk?7_}@q+t=dxSS$n56iNu`9|^@_gyN_+IEQeBz(;fAHscfiEwN7VZf<1daGbI4tU<{_3`_ibROM>8Nvo}A(~g3A=;5^oUXUR+DY}tgkRb^$2GFO?X{I1MV4k`}i7esTy z5huGzxv-AzK#!pZ)9vVCpzttgtjzqw9Alm`c1C92G5eV=jDy}xkEMS`ja762Gac{$ zV&Af@xW(K9u9y?J7|_35^FouSF=!*Tie{%KPBWL=&dN+MlSpqw1gpOKO6d#f+>!6e zo#pcKAbG0H%L`>yx+aa5eu#a<3&IIupD!E zNgt(!l8V*-WBH&wM;;~5mdD9U6+|q?AVT$)*~5ge<*)*3!S;b1I?@s&Sho||z=halP)7Ztd{_1=kCb%fq4Gxg zs=NeMcW{3jE07boJE%Os`$NiXtorLK!OB~#`lrcL^oD1}O`RPOS!MS13!xKCPb~I)_e zO)`5lV&UvIy-l(oV=yw1aG$A_CtQBOv>|$nJ0z@qC)f1)7VI3sKRB z7P%v`!;bu7H3HT)#g}U!dr!!(nJDTZ;_nA>kZ9@{WbQ^_zsn-5S{)Qr2PLiW+!Ps-NyveWMjm+}EP8v`36e*u1RBX+zA`vX;Iq}<{a@7*p ze4>5XD3Wp2LsHd&)fn)OWF2Eb6WQmf$Cd1oCi`9QfMa>cp^!}Zb;$f1r1b_T`CUZ` zo*X~{$#H8jCOtAb0lq!HPT=?dzXEch=2vj%Ei{TC?G17%Bn$Hf2qn8($uEjXCRIdB zGTvkXTepy}v-`?U0JmA7VH5UQEWr5x@$I~rjr;MiYQuo2)hHu-Y>!~h-w8fc^0j{x zyDtypOBY(@s7J`kl3ffewMQLB9Yln;Cta3m4?bz}&Wyc#WFH#&HRO4WYCA^df%bpK z-wYf(uy>PFk4R}n?WAHgU*4&NYj&$<^(&pkxT)jnX6!t~-VfNJzPJWJ=a*42$W1?> z@4)KZfugxU?m8wnUk>YG|3w=S^u2!a_={od()ZcVJMwZhl z6Z@TNsFXOn3a_Q9o-FS-rlwMkx=n4OMWtA-LAm97;tFYz{GB??%vLVbr)vkTP)nV?b$(o(iSZ0ILMm@tEL2l^0Qia+t|EEr+o2V^dwM_Ipu0G~1=)e9t9C+|gMc#LXLR#F8@1?C!cLh&jE z%08j9__H)!wn#c@hSX6FQzj!nHcE+-kBIx|wJd(oF7Ebv1WDN=f09qC52;Rwe`#qW zvVD)4JT{&SXL6Y`>@VDZnw2!At-yM@Q%qIuVU3@Dk6);9azL}dFM%V2-UOuuYt8-5 z->s!#6{5`1f$_IuvqA<3YP5aX2hO;Hu-w0MALUfY3D50QIJJ03er92d!X(G|(rfN? zehm|;(`t9>%4!!IHfn}3m+2`=4`rUTK&a`i;oyr_m82B)C^r0vC}C_N?wj`J-crvC zF;=Y3cM<=Torw15OF66CQ`B2eM@Q9?S49_#jm7s1 zOLCgzY|HDKSLMg}{5iIkqK=MamuUz1*Vc#WlGv}zbEyd;lbt;tXQX|MbD(`f30F9_ zIMq?beb3$3dy{Xayg;UH5ACAnup_uOx|#;9v2l>eS|PM$#KFi%F|*_5CG1UVnlw2n zG-+`1*Cb2Iobpx5Hj5h_cu{BYYO`0S8y~ptd=3xwCjPjd>h7w0$2ZS=!i+{U@q``nWtev|4bmt?K{l6|fD zQ~z4KTmM6IMe&Fh&%2WIg%|VV^0wwRDA;N1RNAOS<4CjX_=!SA;fwT+de0^43N@;h z*S*)5M~o{~ALKX7znV$4l($;LmWG=m8bzLt$_wuq85Efs{VA?@+`5DhNpH(FPEHGZ zhz;PYa#G(eeI(vrec$##d2Y?DZ#(KZDH_ynn%C@BuD)TTaj|KgsYYMK`diq(u+8DOqVGfxi7p@4F7AAC)8y_IbIQ(4%}i`>9;5B(EzOwrc<$YQZ@;|$ z^F8N_QQ7^St321KXJq%$AMN2}$;6~OWwwV7;|Te2gV*H}aJrSp}#{Ay1b2VL@_V0OXs;=ZNZOWKzV zve$Pt5M~M^r7KcLsu~l)_0hcKb~F7oGubuTM%)M8Lctk7c%` zT&~!;lD48NImJ|)-~YYEtBbdjE^a;(emVI0_lBheKIhg*Ej_nvee;5{^jTYT9l7^PRya<2HGF4zH1-IjGmY5I$cT^7&CrKy zj7()YN<88jQF^`DKgV0}Z{Fej`}sp{Y0eRDf6*vyq^8haxKG@#td@&mk7#Z(Y20(} zqvoylvVNO!S>XKO)R4iUbHnF`W`;Kpdmk|*d~L*~=mpX95&{!VyEb{V;Gf@y3>LDDw4)_XeXOuFd~s=rQUh7mh4v1m9Sb80 zR_4bQyezs|w7~JvwNG-1PU)juo7sZ7Xg4D;jWwn0Vy#6xM$=svtd|WF{R{mw1Lg-0 z2uZMx4Xq!Z8*wJ8PSo9)tQc?Hc37wm<=>}K-Q;Yh8zCwMc2Q!nHqs(!xndYjlvCdCl+y8!0HIpTzxiur~apaotWl;^HwXv6C zy2PZ!n#y!Y=94;=I~ljlx>i5Rw&!D~hrMraxM8}s`|jfBCyN%!Ck(>_QbR8#Bu6d^ zap-#+jmE)#FR_w%#2#gKOIJ%47p==*k~i)9?r+ANp0;>*J+X(bS73lC+^F+kVo2sZ z(nLoayUQWkdpRbTN>0U9)3MvOuc)-pR^ll#+jct^2;Y#q{)3UYPP+RV54(|}nJ8wV za#y|~$I%iS$xh%DZHoVhpd>S8SsMB<^j>&rrNAHMr#2ifsN*Yq`P?;~G zG5%e=)pITH4?SFTEA&=k`qPh7ZG+{*{!I9rI9r)jQL^=!|DT3%_E%+sc!kdt%-&R2 zZfRsukAgMX5#L8;F8ooqV7v2!R9@FA;9JnMpu#}AK}%ot9<+BU$S(M);Ln20q825e z?0!xTk$D06sJ|R8*Iw^c$tK$=3)@WdO?ym}g-Ada)?AEw4LfF3RDb{7lnt1+r!(3^$njF)g2C8frDjkBnKO2>cx`Z(gj<0sM2 z4evMo*zVQ`q=zt(y<_{tcH!^&4}WaRDax_r zf3dlpjnxC1L_epIHQqC{)|iw}o*MS1g>eP*@&gM^HeG2S*HhOk&qCKt_d(ZoR|l8u zzQli%`@@$zq1~^I)~0dg+5OB}wLlsqY?bFx_cb4NHw>JywrQs|&(hfPPe{{{`j$49 zu+Y5FEs;ZFG84oyoyvZ!5Ld2O+-;*xiY~qOb^ME_5C40(;nn*uL8Ws%fB1h5-XAk1 z`cmZQkbwc0HIvmSX^=NYc<4RoUFALOp6h;FA{5D4#vD!7${*Ie(T)ppJ?(fyuYkIN zkBwH{Q1u^ol-*dkEUzFxvmn0I>A2%A@N^NPy#bzD?v>7#&TX!n-dXY*ie?vS+v?XF z9&1NxIxttTgUqUoz&@z;T)3{U-yZ>VaA38>jGX3QIr!{<8Zk@8d-oZNFdk{*VV7KUv(NgCiW$*p5nSlrxmd&zhH z{o2LvG=u4P%3%42)JrW#J2n1(hlA#uM_TV&Ux#+F z+RRnV8f$!LT6n{-xaf#DSDF1O{gNics3DEC2G6w|PsZA}{ogo0efh?e%)SlKd#3X@cBCOYaIq;rpq<}F%~X1T*uitmJJnM~ zDDu>muPFnR{X%{32*;fwTV9{shk4hFezPz0aKYh8K z8Ie<1!oK4-{ua? zy_UDe=5*wH$H_6stHiREnekFMf7d>#_;$grqMOCLY;~QN+(V?v@@>U{ocAQ@5P#V{ z-u=n*p1&ner+Dm1dVnz;*B{r$ai{1()Lr@+J4D;muU1e^b4+N1u-joZ!ZwG6TWeV- zn8#TzT0_DUV;9G3lNv?4LfiU%P-oby7MOl4%3S{aR?Zge13F0gYwH*f23!ru(2d|u zsHenT9+%^*Yl`bFJpFB0Z8w$;qRrK>R4AHIur+^f!7STS=Rhu-XQX!>_KbX#ZeZ75 zT`7g1<2~gW#@_)yBGnnpcy5knxpsiIu13$5=N>RNrY$#6yV~G1)-h!Shgt4f{4FWg zS(ZW8U(AsfkEMM0g6Ju+C2@Dc-vt->h0!bBckL5x#f3X^yI|+UQ1^GKJN>IJf%{E+ zk=~^CmA6R6_`%oTo*eH-eut!rG1A|HDpd8nwAZ$cv~k6$C7o^W9d>6ruL>q9S6(3w z6n^$ZyXQN5+h;mE+8wStuG>OY;h{vyYo#PSLqe^ zf<4VvVt-~>ZXj1)AMIxgNHGP63^M;@PPKG24>L37b>{QtzeCH0caFIqyCUvEMEBrN ze!bLQqQCuweOY0GExp9%TqXpFtJ$}Vpc_uzVy4JR*lRRN7>?a4^Cg?mK;13&RSxkJ z`Dw0Q_5n7XZC^?Mk~;Pjr=K^NzaXYbR%CL#o`-I|v!C;+{k9|8F~r^6JAltZPQzas zfL)FO*jrKA+d$Yaq#>7MWh%1UH9u?aaz^fF*2*Z%3`WOI&{Q-$^{)`vJa~S{Kjt?f zlzC=Iv)~=U$3sq9459wvqobS04vf=AwzhoKMbml0UHA0T*0!IW$Lt5(UQdj=6nnOA z(Wljq%n><661)=$E)9R)Q`NiI!+65^1FmpSRY#8Jt!sv74m^s+d>d~U;edN9 zueh&zCweOLo%rkGS-A%6<_qk_R_PPSk+h^+(G!`z$dOdmoYVfJ|HIGa-y+Z%u)tK^ zRNqt*lpiw6+}4^M+9yni@QVtFEr=)^-N#aD8meik6z~f@r`$bk72E^e(c);nnb=aE zEw)i6^L6Ek;xJ^^Dk^1_1o^DeNOFi8(dK>OndJ&_jkTLy!yR|sL9P~_E*{B!)m`N5 z>DIbGx!*W1c_w>Sc`v(SK=~VY6ZgN~49{2hY5qI*g+1jZ@vHbwlBHqDJ5)xj7XP;Z zdv(WPf9rB~Jr}Ftv=?>T^dtRS7WSldyVJ!|O%%h7M49;%)A^;ce!vihq2u)YZ+?4*NO&avk-Y_vCo4cptj{@+7<6?y~M~ z-mTD{E<7PbHXPHA#-8>n-npI+-h)DtI7j$d z`M|fwzlxg3x9}_#mMB9h{L(-Y)Vtzqc^|((oa61|-Q#WLJ?V+`UKUD(q26k4lV^l; zg!72~gsa4H&^6IL(|yWS%bo5{gtVr)x_WG$r{0Rf@32#XP|6P#^97q!U3?;Kkryfk zdLA{MsZT5Pd#W1Zl7BLH*&&*6U4-9ZzmEO~{U;h%2lNd%6Tk=73N#0CrlF=Eru88y zRxy;fTCj8Av1zE`sn){Wk=^nYs;N@V>j7?R2p-QLij|LYJA^EGiP%Nhir*8sC0e*6 z9`W|)zwpiZZem^0U&h`?X%L_7?e6)<-OrWl?CeeRY;at5#lk!K$1%^I?2K@Bfmc4& zdmUfw7yN}D{3c-tpC*0~d&-njqU^_iLFqu7)dcFhI*Hy)Iq1&pWR_>6H2XDK+^^aK zeHTL~!!Les{bu{6_>cEbH698W74ReALSSsr^&o4o5d6j*8@$}qAz*V0| zR`+T>Z9R=V|9IluQ(W`iXB^`k(;O=twVg*?t33v9syE16->dgpyvMw$LRFEKAIe^3 z4Alg=4KtOCUlq2Y?=xA9lYPom;Vy71*d5GC?h%)%eX1FvtDrxkpP+Yf4|TV-qW-F; zyy2iW!w~OxNw>=or0eNFOp_K6XDA3{umLX55UIbTZJ>Fh&CpEH_NVpi3F;gDiE6EM zm)jwG6E4l+*GrYL>miv>;Wv2;-JSVo-dEn2-fz4~^y34C1KyfE?|s8h@^%+;`7Xjk zAxmt?5B9eAuH>)rTOi3ONh=M*-dw%VMP90W{3{to7wWv4voz6ZfU%8Fn z986`Oizl>`IZnHQ>8n1Wsu?D0wrCnM5lp78mA)LMDz`NYwHGw8$b3DcDjNmruykB$ zpFQ6G~J{JOt>LNXJ+p!12tiWs%$;FD;1>8kcVI%2otXf>6qJJV3y0}(?y-^ayOKk;{Qi} z_vDI4rNQDl@s^T{2+21253!xJONgc;r4O!HE?Uh|=JI>hf03{Bi22@b$SrSYAJO~O zACj5VDgjIv=_A`uyG&Fx3z4r&WS?m6G5h)5hEVl2^4%)^Rq4$<&~2l}$@Q7()F{ny zYQHv-D%5lni~aw{(K!IwwR~OtbRS<1GBdILV%zqMZQHhAY}>YNXJXsQWN>}h-QWIy zm702$s=Q?8+|%8=_gd?>+kg|jid@-QQ97gV7Z%d-GBU^G9=I4{YB4$$`Lzb99~v)B zv|s8b&r7vw6DI6UHOs37^VT(o~-f9>OZtPH?*yG)D88a5jB6jnGwmeKL$f-UO}X%SUc1hlK$9 zV226A)urp^Eape^h1Faqu_5kl71iG>Uxg`l#T7$e14jU1!w01iogSck|Yd#=NS?ihtoYb?kw>@i$MJ(GGV zSUhHK&{7=QF4sLP+Dw&O(mnbiJdV4I#-~4U z3g@#nI#!b|@M=<8?-xv~nz3~{>TMgQ2vO+h$`YVlRzea$N05xk@#=}XrNgO%-O zPT`UMgY6Wn;ap^oeG9MR=Bhh1uUIC`t8Li7oZC*~o}*>n!oiOyr`=5K!-dHXnjf^$ zGVWZ#pP?K|qGKd^mvTm)ug)jGBCSYocYZTBhZv6oyxYoS3}vn5%`6*1kdisd=FoN` zZ|^pSqGi%yvx#slvd!}}vd@ZysyaKb7lebGH_p- zp!lFZpKJ_QyzZy^j^JDRPWdBtwBCo$iI<(P=vr`T#&)3`-_FvxQlvKam{mLzB1<8AbPv9I;xuzUe+u*&51GP>=~LW5 zchXLyeXOHck#9qHr}gqSK-q9se=gy(utSN_ZyC)HCzixBwd3@**k0I&XX<}Y3U>~C z*X^}zj+3B!zSLgGo#jvA8(ce^#4Pfq$Go=J9Otaa(xC-resQUoN$V&q4#kM)aZPDx zBp_x5rMnFOk5)k(B=ifmb(FJ8I_3p(sC7{}yD3>lSMyII1-*aG@#1@b`xxrKF8d;? zIy*GMX=|H>^ZG+h#*~k+QARYsR2s)_*zLln9lvnU*l&7-DqFO*%eCW{GHYpt*1W*4#7Sco#JCG^95z#7eOfGlHy z@FVwmqXj+;yD&~37`@J_Pe*dg0(0W38HvL4-*2M#^UKA$e{QQi_+;{sPK`t>iPAXk zl({B(WK=Wmu`nypS3IGNL*FAgaDI;PS-F&;Ev7@iyq{7kV^72z_1=1Ee$_^Zi(;Z9 zS*`Lcqm)lrW)!mb3n9KLzdLxAlN~FpWRw)*)I`u9S6HdOoUXl!Ded_CfsB!RMcvd# zCDj$gm^~cwUC?_eV;sYT(P>)bU|enfXkdrlO(^VLg4}6^n9BDQn*|ngpUh2B9&V7m zICY6*y6~Pn(JJz_l@x8Ov!oWM9bmHKCTe8l)Q74$X#w$C0GlnM&a-;-M`RrQ-4C?9 z{3ZK1JuKz4U*UUpZqZ}6)&DpgHc9PFG%FW^mDR-qZ)r(*7qevjPdeoL>=rSKFh zuk#>RlSXSasFkJP zT3f9yjzy($Yc*%+F6%E=SB8a`+nc2EsIl0=#-SSAg&6F}^|xkVk($>tk=Fflo==V4 zY7a7l!Pnrwb9s7duTyTRH`P6I8((k#I#&mAD_1R&WDm2qyI0F6{h77&8Tep%afDez zzb*ZcV&HQ>B(7AB26{X5gmd%v#h&b;xxsi6b2<24fxEslhp<0fUsx_L=WE+ZMmD|( zYoav}(^izAcyKZ0>) z^e^pqVX-mPQ_+wf`zrM@@2V3+3C`n!2cNNa8w3^P z|HX_Y1vWb$@MHO)G06^z5&j<~keZI<5wk?c=-){VeYR3tXhsxGCyUe>LVp_8`&)OM z3zQd;zbO^f_3}ZnmDV~^5{>h0L^btSv>du3)9^e#qn?vgRZ0tY?Rt@i;&-mAa>2N3 z?~>B<59}Q5xc=2DA-^G733N=$5I%Z2_(X%@J4yu>2mX-}_BF1Xv>nf} zM$i_vAkEcAc$)i~IQo!$IGf#%zpcG=rr>?SoI=y)S}>CAa?n;}p)fZx-f_+71kdz5UxmJ~ zYC*-jR=#c}v%T~@eTfq3Xloz2%;ymf(u{PuvISSy>ewBGhVjFVpUUmX93fVl2?^RvoJ1=a zHN`nNQTU;~R8&ah-ZM@}M?!@?WrI7p-{J4+kHB3%!M;bmxRd_bF+x8FrS5jwOOI+L z1y1Zv|7dYyCh;vdCbAMO#a+z#LLH$AxkeL>nz)fX#Xf?&MdqOW+)q@TuBUm~Oy_@g z935tSM(x0%zBfEvs;Hi~gJhENPF3Q$UJ1X0chpSQNnx>`&-zM> zINdakt>*tRo*LVwvG^oAWfW9%i$!&>@0=L#m~GWTEp&k&?JmibR7ORoM4xkK?40~p ztB+EJ^d~jxoX9n0E_y9wgRWg?;RbubvtSpz2UT{|HNSAPg7@5|LiZdgW-&I1t1HbC zgK15q>EP3RZ4TfM;BQ7RAq>e%7b42AzxAU$}N1i$VK^;RLs~EZm0~zkHl5_9(%a6Rp`5;wV6e@7`)>=ByKd; zY5&O+#G}GNW2pAol9W2qX(3*Fjz61%`dsgY^H}rMh}KA4%0-h5{0P3N-i;f}zJnvN zkZ_EBuv(Drbg1`KNJoPs^IaK@wfq#TI=>Mu5}s?nO{aJtkL6C9UC}q`2RDc0;uJj( zf0pZy_Hm`yH2J=jiL|EkX=OB-@2_0dj+^ThKkscj zq-uJi^qoAAZrGjpQ`T7VkhRCIjvpaaI?1J)ZCML`E-H#f(i@RVQfo;dPp#J?2Q|np z{j3d6NueU^faa2uXg3)kFV|M1OYn(i76Yt~Epq#84Eysbimbo(lQgL`w8=yKHFo`xQ#- z2cfjZOJD72(my0KYYcs=^r$Od%^hONXc2qP-_onlqjq|6DBCSK=wq_i?k{ySe1az! za4e+@h2F*yHUZD&2Uq;BCB0DvG_iGvo_9BXl$Wk_#j#pW`Oe1r`>^;rDbG+N*uXU+59> zmi?5rg1r4>dNuTm{o1a6^%b*r|8o-S?UjNpv9Ugp5a;x zc|wbYdE6#`8W|2$f^>FPRzOI{W};-fme2xCph)u@J~2tk27X$P*#pf128n7SdWrwZ zKV*%~&qNna;hAJ0xruh$|6-Bu#;wULX%`7vmF!_sKfH)cf$aVT{t0&xXAk%1Yl{)# zu9ad=uCW(1o%XYaG({5Um$6X&0~BWrlFbe_f!Da zdpo`wxH~{6()x-Umive`4qvg27N{a<-0 zE6csIii!2;b5xa1v+J|^)D*qwGBvsOEC)7#=CGZWK_~EgG82*yt>CSFOxs(V<(XVL zyC6=-HVX&s^hlv;!c(}*-nGBN$sR2~MCGl&_DWDiJJUDzb9<@0P{^X!$6>RdFiH$^ z%eDP{cicz#PNtb9_^;e)_L)myHa{J;ptDd1(x0Bh_o>2FVoOD8)#biIs=oyJNy@P` z__#33%0P8sZ@5uKG|EnJet1ULt~+x6{qXoWPC9)uoI zESJh(#gptM1hX9C2AgN6NLJXpAQM2^LegY8n?@}<#ombi=i2T_GYC6aAG<1RLW=!6ExYr47!*F6pV=Non#O=eU;5|#UYlyjNGVwsJAP1j{yzHKx z9=JdG@MW^rJ`FjI9Q+!?!$y%r$YW4+jVxv@A=xmSdaccL2;{mOL-#imc_yx9XK5!Z zL|zDM`HAE;3ED^a4RkTxLJqLLoFojO@zx5i4jIJ%Yu%ysPy+9PT_}|jREFLp6(JoP z;!c8_w|Lu#S|?70;X<|abtiqPAD#NUdVvHk>evv5z~eI$|s_$#W+ z4Q30hp13@B9AT@L5RdC{&8#ZWL0-alf;7bpySCU1G93b})@-OTbj&W>-^DZ>jV`f0 zs3m`qw8Z=9AR#O6F1}=2xCmDUjbY93O3o=(#o2|Xzz?a(D#Crau&@>#MQ8bzd@|%V z8=*M708$)F=mD-NJ5If%4N0-8ky7N2^#s!8W2{bgKQkAQHk|fotB!TpoMSbFOjJhu zzSYup+PCOvG7GvsaV!no#bap|a9+=)PXPFTgvCSJt0U_~RqhO1!wqCRA>+}Ao@H;y zAvOW3zL)T{Zql>#7pU%2_zczv_6qff^u{4>8o!ylhBsJO@HfcQ`CgCLw`+&Bx>^NLhzQ)) zDnoVH$PU`o$Zzrl7(7QIeL4vZ1A0wUyc9B9pHM}-9ydihAup1TTMfjW$*c&O03PMl zkT@!6^*7&HHLTutA;=#OwElsw|E#li3p)ZXyG>*!y$hY;`QXO?hZ~MMb9=Z#@RsJn z5`Pvg;Qj+955~J7sn8Q0gFUG#?f|KSvotri3qeJWU(umlHsBY4>c-A;X{exB4Z0uz z{t(9sd-(!L5ps){r4*s7kX{)f*Au5o9mPvx3$=$E6S{AR5Ol;jqu;dPc25rTE577NM8>K|A}nX#_IV&7ph?GunL3A z{3Xc)l%%0}HxJtkKV2v+s^U$dr7%?x#mB-GehqMCrf_3m4n!dqhE&#g^2nZG{V-=6 z&CF`%8$&Q{qp6i*Xhu*!YEHG9+ZAmOIYK+pceE_LKLH?ui~)vF0?LV6prgPK9>WcW zRn`}Ef;)6;Ru}e(A<(1zOerL%7PH4>8uVmyvxW30Bp1?iXCb3e7YH;((KQq;&fv%J zxp7-DTAUybmHqsBb)B?R-6+;oMoQ_GLCR=#yb@3d=o*#Ol1id7*YVxC*ZEUzE4LBX z^E0^wa)4Zi?`RwAX7w=6X{#baWI^cPaA|G4zDPSBZW%fdcpBUkKC9&gr(zrPw^aaC ziGGk&dr5b3HE|U|6AFluI7WCVbP=u#mf^`gi@Tam~nNHi4(!->hvGH}jjttk{se&8dYE%sIFsoUV`j8sl2PZdi}Qv7Nu#|U)=kW7|4r@DGM6<4gI zxuS{J5wHVnNV%cT$W3n=$y&ikw(zmwzTl8BuhrBO!^=aZ{fGUd13$wpBk+3zTQjV@ z)*O3=U6dB)g4hz)LH=>Mcv|c$o)O=`P4}TRK;oolLIB)r-ylJ@hvj0s$pbs1ebbs~ zHZ)4;O#j!2=&kj^#x--CSjPU5dvo|o5no@j#MGG1SnM9_!IaB&Vp;; zqe#S8Ae)y3QX?(7wJe4O$yn$Pb|Dvm#4{6C|3mVPTq9mk{;SchKxA6L<%bn`i%W-Y z;ZLypWaV!QFQwwj7^$YbSS}z_rHQ;u&7n4O&2n55xI}tw0+=U z5X`rFS$)4YK%c31*AqbZt8BD36>FdQ%jE42p!~(rjjRIq0Nq31&@PmQ+5uy$G@l(v zRzJ}coE3lKinEopKDUt#q#em=Qik~LoOVM{9IJpg<+44K^diql8M+MEYWG<^=piq} zAMhsrJD) zhWW?3Vt=s*+a<_U8s^6FtHiF-IY_;4lJ@<7!a>v#zN+FE zC>C#kq$EZfWTBQ(jxB*a>MW9-G$n`acUDDfp_S3@ZyyBj@?P>5-l|mE8&YuDAwy}Q z@i>H2z%`y;xF%E;TT1Pv1nHGrOv$J0kzGm#^{Vnt{-9o0Uposq@;IETU&-X?>KNq` zT;ttw9-r&7x?1$1^58QVOJKeLH=TB~-e{e}rGgdxll`9H#K>-~mG&#NC~)4l!oMNZ zI8sWd#(Hp<^&%sQ3+m_?w41*s9+%I^V`aCzPC5ne5rya0O$v(>ggX2+4w(6LGgLEY z?LL-ZO);ASkt3_NMf+EOtXI~v>T!BKW2uR)i{Q-}VJ{$K=`gy472(o=TR0kcZvV?? zZh$QDOYSvH!gvTt$m?(m-A^dFPJRQarjvcldTH&nEFXz)zZL5!TC=J#SKp)c)-GtZbxE(KU(+udJx#y4%N$}2vKzsASwOq9eOxLG z@JEz#6W@>wGaF)l`ydN=>DW@&oCCG!)igCAE`!T{)_}R=OxLsv)md zFRR}ijU2fhIUO?`TU;NVq9eUDmM_cABmM0W^cp00i`u{8bZZ`3AAA^s!_a6Wcj!S!}{cD^ym)NnSIP|xU(D6{~J{6M0De#=uNp+rJ;G+tcmFZ~{Ld z&p~ah08D6LS92dwH|*xWz~1y)SR$C>Kk{#BxYS10q!#iKIk!4pt?wM@IHTTF{wnhv zS)FSf?HpMg0Y@h1J4X}OR(E#KH`fl;Bkp3Ub|t$#HxyrjZtpH*rFJ;HC$PZZFE}O= zt<8#L2;U2&`8Nd$g@WPHkyPzA_z8#GPwdHX_v^+a)QKM^HkL0*&)|%^C*2lbi)E$8 z@R?N?)A-Zai}JHO;9_Y>I@ql&znR_~Yz)$UTB>$Q+oaXgSL?Zqt41++mu^_~?S=MG za6Sx##N=<#uP@;Q+zH*`RCE#Qvs(OgJPX)rLC9=QrjzJII+~UM2eOAuva|hfhq6=b z#-t}{LY{!D|eI*s-R|b*y?Ka zr@GbA0jMO!9LTxCsXGQa9Il$4;@;Y>ZORmJ8QM;H@LK6yY3Q$G<~N{9u$T0sn@PY{?7LP2^RT{IE2DkY67;5eBmJ8;RWD_v8H3E>z~Vb$ zMG=!speI-|oOK~I68! z_IA8fN2y;Nd7Qr-M;#L!8J$(#dpyOwEnV@-I({&lWly(iL$^8_3FL-eD{?1T+~2@w z`Co+BYMr#lp+kWezIDDLfp(z}kt_OYa8C6ibL}2b5B#P1Q7z%1v|jl~9jR=Rze$ax ziBb!xoZLXJB~2DQd~;NrQBsanA-nC4R!{Sa(NNE)HHh4ZECGgH&&Xu0gWdpYf*JZ4 zV~knb%4Gir#@a~M5c2gIxh+sVd4R;(3X9N#?urXx8TAFqWnoB8muDKS23?1*b}v|Q zmB4|s&Td1tlW8Q??ga0+3zC^{xwWVq+?K{eGPApoU#ulvmv<>kWJ`7{XVqb94M!oT z>QY^uU3uXtJ#*}IZgZW1v+bAbhiAR_vfGwFq1@~YvA_xahBLWESSc z?eNP;zetYI5&x95GifsdG2yq7?s{$WnRVQr1?NZ%&Cb<_U3-p{Q)#VIxt2Tx&i@Be zW4VIzR^BbO7rNu7+&kJFh?31n0Js@*jn#S&?MUQcxJY8728J6sznddIcyR)6s6~rAZ^=L+y!)f+uI4L;at~l;HUODHu zuDW`<#<Z&~&CpQ$xwVH?L^+_J5o?UnN`eys zbX1(mCQ#q%;5l3jt4j~qSFAtgNw~M!+U`jCaQ@K4V9wCF@Z*R_s{%g70Z=J5HImGZ z_7hU%|M}1uQtfNe8~hIL>eYlFd_|!b-yhPKfOXs+>fCy`BVSX95^sR7bfvrp*3@V9jhf!I+O^3gyQAHsJ;gk| zJtIARJUu*_y%nNM#@zKTa`X}Uvz_)5`yJbdda{<5p{WtuUoUNAs^QxgEF5CNPyPpK zuTx2C1OJA=nvh3xn|*+Z5(oc&n7#wz?r5onvO;~J_EkTsmEh)FTAivsQ)j7dl}XYf zArZxKsdNHeNNU<&%*AG2ql301JR;OQB*WKdAc9_tJdccvR0EdcD}96U-0DlKv2<|i zCBi*h!WZ#1oQhlX$N6u3HvS&2gvY_JD7*!I1+w%}xQVo&ci0;s_0zk%<4MjeClfCIq#|Djd)&rH+%W0Byg`9 zF~_41yRwQr-0*jh2>1zZkPqf(ZBkhC9ZQK%NlR0M>w`rD`_opW8mW5P4Br9&te~pR zHI7>|?ee4>ZN`ps{c%=MigG%>IKHW~q0SQ3*UD4{t6w4YTSF?$J5f4r5F1aMlegAv zaDH#rP-J3gSMXtQLTFX!MQBC%YPd$Eb)=RSuNBv)0%0-$#N;`kTzq32xObp(NAb`2 zzQRZ0tzhw;@h9LkP6Su+K)eDygD%T#dV)kj#khv}|DW*^L-s@ZH805pYTzF38tR81 zf?iM;w3K4fEjgolPo<6>&R(v!|CPyQ(L*_n8-xTQM)do!vH_H}-_5wfvW@g*qZT)DH&R1?Jl3kvqN~ z$v2b4RBylv`U4q#e^b^Z2a~U)%t%`rcpCX+JhZU=$KFla(I2c1&LAZ#oty`p^PQ(0 z6V=PGkBm|r>JqhpdQko@yh54TBr+1t>6OqQ8)wxuu4#3{Cjwvm*#iSZxx;xPV?M(aPfm?b*_G1IcQYV`3I$HV z$f!%nIa8{qP4Kr4BN*@7hm@zkfP{FORY4~OPU+$}1M(i~3aF%4)n zy@2$S4{b*ZZptI@7ylH$2;2BDiUtnTYg8DR)!#rDxW|UlvGDHYBZz*0-t1gpU@QT) zP(N0UYlPP06Z`^U0dSCq$)jYi(o&6fHgSz}m2jVRfAduLCVM|c)rol@qo%tMcO&j} z^eP|-0OJQL=*-+)pvLsFOKT=b z>h5hoJTJ|XxENup+|F^y)!V(@mFgI%W>7ZBFXcFOhFVwoDDJ{AkrevNkFCk(Vq?59 z#JH+8gFMa;e@EYO|Mft<(57&<$iYaKNJg!HWR8}}$PAt3BFKxn!6dO^96<|kXW?IA zsyGPvQh)dvu%8yim4H!u64$~LpwHrE8|h@AEM;Gmy*M>e)P1p;^qYd~s|4e{z zfhbG=$hDL~>OYQwu0;2GPY!PzZ=0x>QO%==#vG27<9j4*j5`}uTb1z|AXRljAMj8v zhiw_{Ba3}&{@zH6PZ500{j>cCd6A8UU@Ag3B7PhZWf&cJUD7xGP)WG`jN=UP~*U4e_#KR z!1KV-;Iwc#?G+412!+muSLsiz<}f9oENC8If%Y^8cvNR0Wjjbb2>G6q{CPNaUxT{R z3Z8?C`=VHs^?xoB=xt@CSEX?!**KS0?^kyd8VM(@t&4Hvsi(3A=<~CWE3E@`sd~Y3=>K&<(x5{T^z$fxv7zfTH-4(nxiMYk>QqYcde(b~`4i z7ZgEtsza3J65RE18}ODqvvxyI@2Ngb-=+-?{|XfJ-|}_x-}OfYr-gEaeW4?vD7as_ z^buww=xaWp-(Zy#0?Ji-Jc2JFv=#mnDhuOzrE9=k42A;$(zm< ze>Y)Be7TrI&VzhA;NtAZEx{#Mh-R|Ww7=Prn6jWbyDh(YiU81|EyQ8zX;{cG9v#Pt{d=1J)q!GM`)`F{} zuQ^>Wqkq=V!aeh9aIF8DFOzSTuYjNU+XRP%GK5!#zlC>2hCtV1irtj%Z1NeUq+d{I`Ui@j!=geH5cndTtZsvZ|x` zY&O}Z@An@}DV+QuxnOEGpY6ZoE0g*@d06u2dIdRvy?>!yyy2itCNW;d^)jzn*`1tATO7OU>YX>fGzv>lQp& zqPXZ9v3#7Ga5w#y^yB0FUS1smwA$$`jP42pgj?jW-aK@~H!<1!H-BnQ*uiXnH(!p_ ztiWAZncNlVp<}~7{gl-KD8OY&UFu`=_^XoBaTRiy*WI67$DDD{Lw*2jVW@ghQRTzJ zFVv4^CuhwUMkk}Naa!9FE*CuGI|s-$U+Og975|9f$6&MYjqtnhj>uhYrTNnu1hWyA z!W4%-IF8TBH|ITqQ_L+65UYuE#Y7<+q}ne+-Pr+Z^3iA@C^eg4&cbxK`wslf_iAkhEADAzg(hu+cHwRp0IO^z^*(9*K^`luB1MzFPVt2}jaB_hwes;qyR2 zEDK4h0NP+nTGilQ->&4&Nf}ed`nw0C0?4-|wMNRiuVVEi8t4y&Vb;}T zbWrd~i`4_JXwQE4AZH!NXeF!sQko$*P%_B(g%s3@^(Mot1mm@ysK0?e?V9k?V4^R< zcOk8!FTH<8pkKIiWEHT!o4`%2pwZtdO4_ol=q{vaCZUG-G9H5O1G(@cOn%r0@4{

B8`4B>qd6?he7W8!5?38qriaBtw1hc z)-)q^MrwcGp5SB6G?$Q-EEj43otl9_wcaQ7RZlx-yI*_aJ^5T^9eI?W;wI?e{S~td zC9nehxFpE5UW(iZZw6YyPGIu43RMpb_x+bvz_$!I(+jn~`V4TTbuxQH$J(}b(7}-3 z*^gV`2KWY?l82xdv>0UvlHn2zCj>hTb4D(~32>jz1s%5AJ5U3>;-Bf91=s) zB{@aD0_#3e`Js+-&T{)aFT94Ac)vt-jVTuQD?V#_l>Sd#ns*9#aOUW*G!=3S`)Mm< zMOX)N`}bd^es)f}l6DG|9|wG=`FvK|0so`m5p9Eg1recy^a8s7m!W60KJ*M6 zULhcT=7mJ!dQ#f1K|o)|GkA?3#Mi}TQ7x_@dk9KdD`56rf!+NWcu)nqja`)Fg#LFe zU=F&$M>ZDL|6gtuJ}2~+^Qko*bwK&c>naHEdYXHk*Ad+#rc>;V*pzhL;`${#NuQiy zMS?H-vZDw$S#KB?VTMI%bDjP)eBYNT`TWnJ-}#@-{yy@T0jlom(2UUG;OpS@a6#iY zIfq*VmuI>2l5>zFB9D^`IA%L}*DzNN_XPKO$5e3nb)=LuhMVOxTQCb7ZMD0hmHzI& zKfVlsQ$bhQ5qTbN7ETxThdb*{EC-zi>6aL|F_pqQxVu1z>xHhMQT$n)f?7gePq*h; zCrs5!1R8uAs92iy%&ca;vZmR7do9VqI>HWj5EtZE^0UQ4a%1(8lXqwGZ1RZSa^Az< zq29$&SEAR)Cd3tq%OAHRerfs(8EPcdiJk7gf{U5^{TT!6!X33yTF<~^-@D|(e{z01 z_vPU4CcYw(nc7f&k$y?L8QH6UH;2>SI7a^MIO9IzuIDVR+R``qm)rq*(C5`&4wnL_ zGD|d9==rtIpuDVsG}^()U`PTQkDre>W3irpFDg`Pu8_)7d9 z-i5Q_g9xF%Yz^%S-IRaneQ+%9H?|tvfkAu1tOq@p;dZK(W*4D{Xn>oE_Vbm*rcycS zs(25keEd*eJ1)9^c*aAu+b2r&{^#+#e!F*h>qo7NU6XEcLgj=%|0DIp_ICH-1- zTB%EYBO+PCLw#F(z0%J8&HwH9*LA;Ir#%gKHhaNSC3Xpzq|lTULm_^UB048{c6ipi zPOFcBfZqyV0qSc3el%#m%kWtK z8Nt1Gjo#v>;~t8~G1S%0<%TufjNgUIaCvAt_-=1AKR1xd_A2dRYhwaRA&HP00@+G-pGV!3)$5W}fDU2~iOYRkT}~PqdO+wa7(%mzmL8 z37=0k;2vDo?&)uh<3QlOYZqnp?PjDA=(*XfOEzL_fW1~0e%6d^ESCsQoGj!y8Di(M z@|lf|@A%Sqz^3xr~JeudBZt39f6Nc2QE)K=0G2&oPbT$0uxw_r!LJ`mQG8CJaYr`^x)k z`V0Gd`TS|`pjw^p>+e4tysa0vGf^EC=BtR~U~l>Wo}p!+#^vMUXxMHGbCu2;vM~kt z+nMxZ(EW3nBduMa?**Zvsbf!}-Jv^H+J0*-f?nfAtGa!H&H;tuCMXTPz>_|d#*;?& z25W^y&AO&&WHToLYvQjN3zLb;lDkxAg@Gr03wj_Up>HVhJ)l2-86V|;g0lPooG?Ft zR4YJtaj?7*Dx^h9H^>1!hiOOe#Xib-^{it*oG4FSO`TO7>0GrunWNUnroyMa(X+z& z6(%q|B~OhH!Kc9b`4;*o@;%fDh;VnpQ}iZwR-$v~_}219aX6g1L+NVkxOLYkrDf1h zL@fP|kxT0o-WS=Rozr^jIjvD-D+_b^=~`H=m1srySw>k8$bA}*9zxgmDP2l?f=}ZJ zWW(CPf7cUi0Sypb5li*lMm132XWRSWGk-!5D+ozg3AKQkRhePJ)*F}~R~hEZU1LD} zj*dr{MgctbPM#Wj46A%@#V#y4+Fer|ebUsFfXg)RF2L z<*+)$k=~iZ8Kn+a{9-#n=N`~|WG(E4n{h18%?G)^7~{RHGu#?w^aMqa35rv8Yn@qM zOAUY5lEePUERz@~NI8o$XBjntGu_9k28{AC>>qX%*7iO-flRSmz@B^qc&yWb)tZ~- zfW-c4JIUH>PbE!2Id5XWfX}%&tk~XG#9U2M$tMbvoT1az3MMqlNJ2ZntMVVrZ66D_ zauxcVP=x|pwmmxy`+8-do3`dYa6axae#2MeKf%4^8A<}yD7bq-Ri1~Q;cEB^u%|`v zyA2XL0A;$llwQsvImM#jS-k=p-!0w;wXZH-h0dZOU4stH1*ob!s`KP)kkicqJ@nkr z4PSxIfe21M5p_BQiBE4TfX zaL}I}PPUOm;NYI2wc!0f3c2vs_H8@fK4-51UrA2-%YJKaw5HKRY(Gt)4?zde=qOeK zoFF;iv!6^p01JAf-4J#`84UtbYDc(Bz^p~CA1w^0*b-ouub_886^`Y=RS2YGm;wu2 z++M6SIPUVmM2|VpFWrGH9toxR1fdxpD|p500)n6Wnsh?C&!+)xb`$W3%Ww-@+^C8{~!R@dvSR0-qpO$c8rK zD=ZsK7C2;SdO3Rq+apvLvr4tp?7}@d+b&_mnH`|3yB=n~Oo61zJkp6wf_ylL@w|$4M1BE<4_{2_e=itVhA2$#hi?DiuV67uJ=e6J4ag3u2A)VD2 zUq=s+4$1a`B#F&4ClcWG5({pK)6gF_25pCqiNycH!+`)^6#hlassi4h+t3$yz;zLd z3q?^pxHb3k3~eBf=>pD+MnbmcA?RBNX%nDdl*A8#TH6Dm z(7(38(>fkqf?RA*)|-n(pWq3O0+;?%9LJpmmHi~j!ZifvmkI0Q0hsA-0advjcN-jn z*-cJXIgG{l&1fb0$aMz)HS22kL%XRshw)CB){!Xnr@! zg^GjErvP~Ou5(vd52%Uivcr5cn0@O78gWI!v2LK%BvAxxg&ojm8wkC)RLBq9w=FZC zo@3|0M>`u1eU=SL+$VIAFZ;h4srE|K zB8#90+yET}ZG9oA^>x9mwVpjEhd|+O29wp^f~V7D6=9b-4W#~fREHCR)tro;&oYd;=FVeE`k?wF*pGg;WpxV+&V15jS;5V0_$i#P{l6eUwj?X?(RT3s zmO>r)7TjjwoZp3>^cbKSEA#;QP7&=6yzUz`J540fTpVz5DgcRJ;C|pW><&3iHChv< zN2VYX)>0TU06SkRHSpA0Ef~UYt>PF}~+Qc0si$T>{#61MX^#<|s&wvv) z&t8YC!;@QVpQIMa1C;X&;Qo35^XyK+yRx2ZPafhAkjAJ06CK-fpJ1g-SGc9rzB z=Wwl|tHp7b$tBtfYTanQIPha!pu?O6e{nB-9FjOQ>1|XJ58&3@%Sm&fY0pJp=y!V} z`3c>|?l1|m1Io-zAr*jWy#}6rEPY6alPK`qEkXm(8#a;rOBVrwXe-+Q(+g&ByKxAG z*m5eu${q!8);;i37vXx_wb&%YqnnWE$jRz+Il;4@pRECPzXY7&S9t|~ihjTx+hk{e zq_iI#GL3*nULIE!s_?Z~44H3(`vd)AL%Bzmhi&Ex@Sph+s3WYCCwLF{f;&zX_+;m? zroa+Eg?_V{+%d9-&WDxNpJrwmVJc>Q(D-&k&T!%XeCgCc4WTztlj{Jkfd0U=O9e&d z1FX2lu(Ljbz4Zo&B4eqW!2}7+aV9;;dT_Hy*lGxK3#QRjw!{|gt~8#T1|EU)=qj9e zV_-_gLblKLlE1(+is7zs8)#i}(pqExi>eFFP+s1Na#-!Zg`+TrNbw|2qwycXjSO3!-iG2oU%m(uSDAJkbW=`KSf1ydgA- zjV5)WFTDw-m3+sS;iq~F?vv>_2KN`Y0U3M)_YqBiy`d0y4nBuVM1`*ER;WsrqcNbe ztYM@6$32I){Wf|EGc&S))3__%i4L(@s4Pxl%w9sT*?&ly)f?C+N#I`Y!;-*NSsU)} zUg15r9_Sqz;N5&kUjvtWCf+P$5|>K`l38;r%}58+7`uVfzB8=r{8R!mj2}|T*P+V! z1Kh<=Y=lu>zh)70hFuVpnH=0D^qVg!9K_$iGo6byg#BSO%+d@PZLG$2ZFUfE#e#62 zf1(tXn_w9}l^Udqnb|mFNA#g)a~pBl#nr-fX&%hh>&6e2?xKab5BY4G)?Sz%(iYa$ zE}TagE6owx^CR#%m{Bp7%%END&QSLrU@J*wdJBJmZeB6roYYfn!Iu;hxYd>y5(Xb& z)<{3LmCFeJSzgQwI_7gU7&;#RW3@jAU)yjq6D1^u)k779!NOml(3XJA(?QbK^jWtc zwR0KmBa5NJd=BsDB*>LiC#&gK>g4l6dL=8-K!tcI+y{m4HUAF~u(HFaG#S31HlVo| zf~nIg#^?^Lr7N(*+yxisA#k95BqiBM?iDD=UAPf+BW;Fj06BLsXeC1t^yTnou7r8R zylPh`>se8>Sy%{srHN3l_Ms1<78nc~>k7!AoTkU@3$&e;&aQ@6O0!hAqouqH?*~uG zKynKlI*H&Gss~AjCu|07LHEI4{g*E<>|_hc5ttjbkIT=$htI18oRK-W$J9aQ&`BgW zD8TdJ%om-~x<%MhFD`7HT!PziB_yh14YtYuL9aLnuU>d3dMu2jZ8w6(la-K+O;SeA_m&kgnVtmIx|hHypwxP!4gwFjiPD zDCjV1O0q-7!ZfbKuJniHgeN~sSk5QN(}e4dJt~Tqs-Q3H*k3EGN0&|=?oYg z{+mza%Ls{(Al}IbVGc$k@Q0__H|$aGHU>N-rjZYNW&yJm^bXgckudk7I&>$l!8v&r zoU@tW_wg{V+6C_Ltpt$~kkVKT^9%2S$3GRw{VnluE*dJZShR}U&;8}^2vdY4X^h-d z{45OR7xS$ky)qtdDNDf1RtsyaJXBQ2Xd{?w@DY`k8F&M8z;w;sLO!m)5YK1BD@X-1muni9~cSA0<7hNx|?=}sf|Zy8Pp401;7G^xuGSY;!S|vxe83j zEe-QPN}`gGhpGzs$zv!B_6mNns#I9oE;JCmLO^&ZxultRDjv;emX0aMoUyJej(l=! zxrDSxIprJ$zKo#foEJwOa-4Ink_FdKSE@JJ0o#gVf|HTvYDp6GjiGANNG9*<>5;}*Y_!` z)-C*OZaKRKGlmSCqg&0~nlsSKKgf5}7-=rGwy_1|A0rcK#BGJlQWZ3wRfpugi|jUY z7(?}`b}ZaeCWCAF3wZ;xKl*a7U}97U1RxET2DQ~7*y+lGmgr|cxmkP$c)J}ag%3fF zt2Xrgm+`rIi!UG^l3U1c<-$@W@dIB-4oGj6Q80J4jdPG|w!2hJ%cz&pb)%|9e~5V# zyCmJM*dy-ie6-!uND9HU+mu-;H~v;i(vxIJ0JPA**cQj}U({~iBhH!1Y-uUyW^*Cw z^)mb=_$5%w7xa$+&*{xj79*YYh-}2Mpssj@&!QwvR0e}PxtNe1&LR;!V5!>UNC_<~ z=*lCtvfAtLp-4%isrfG{NE?F#sT!O!=|CNv%Pql`xhasOZw%AJ2#Hu*?QH++kFgwV z0G vF)4)vym#m&H4vSa$+7$(Qo zcHv#S(C)F4nHVQ_PHcN(+vbUF+qP}nwlzs6$)IO?8s(~Qz2B89=lqDKd#bDUv!1o? zRnhVrY_t=Y*gs=huWWf|ZH%4sB}YT&Z})-Fm0`!i@`l%mj*TuI(=Q@#$Uz&!W*Prz ztulK4l~NP_3`pLe5=_69-Z5*uvPnC_aF!g_x3(s5E#I_t=Bxx)rdmd>8T{;P9?0&$ z?(G8&HmARm{9d(diFzw~lIZ{*S{;6nB{%Qqp0J;gM;v2R(EEu_DFKd)jsESvepy0b zWw4jnPR>%Il?iHJJ-1%Z90W$zXH(MSjO*G~bW-)koi>P_`UBl(ls3N`Ud#Z~q0>A= z*UShb7u1eObQt9)6_7-nj+AQ!u95IYNEOCgMq9hu2RpXeo?Ds-IfWqn{r9YY@UyUG zrTsr=LDw92;ZPHuqWfbP#4eA%7qZ#e$@-l+BQ5r=O&^zXJUv_L&)+jsJgKEqM|ka0 zX{8P6$?xTV2G3bL%QPZRR?`nZE?=;WVSH}pyjKt8L!{R^L;TU!DgffI>sndl&y*D zVU8u)))DUM#*Phk$yyfahH*k)>j>LF_C%x_N;*EcCOd5*!$Qu7=8m`p@?l7r;OxpZ zr3I7%zP8>8-ujtwnN8EPrF&A3q>ah2`KQb4Fyp2fS_O_0Dj)ksM zE}z>SS|D;&WMWja=mp`;op-F$*)XGo`q-Z<&?d{we4pMCC1T}LV?B8?Qv5&VZ^+v_vI;TT_Sg2LGwP{)vPzCqvLp4^*l*CGY^pzW)1g;_0fJvLA}H2`aIO&+w25pCRCk= zP&Z?Y^Ejh#*1E}@v|ns0P7!CJvA)0P7CVZjJ#l8PTlEx6X-;Vrl_ZB z&-8Gk4pKi}=$?7uH4B07yR>;4p06SBwp=3)W+Zm&bD`PS1mSZhCVC_iFoE>ox^OLp zKX4{STDMs%+m>0sSbN#?Ir7*;?PDE%9S7~X9IL>)AMe)OwcObqoa2}+(lQ$9m;&Z( z`W{o;WMyrTh@nz+pcuBy={|p;m;bzfuCGqk7Qa`lEWME&(nF=ORut~U>hQl+WEh-o zSM!f(G6<>v+dXgMpVK(j#&%}1kRxf1Tw4w#3g1fIm9oJXfz4tyshfBRB+XrD1Sqcd zL6_tY<)YRDyNkE*xR)~)fQVz5B`{|m!hR&kJf}{2gKlOy^ksHqThij}LAsNUqqUgF z%xm-so?uGTo3sLL#+FA*O?J3DW6&x1l?_4CY5>27-OWWJ#oS03!29{Og4L2NEVcBo zn8FevLD`WY(w7O3;D0fkNZo;`aQ>&(Dp}X=H&ZNy~DOwYhuHI}Ht{vM7XMPJ;i1wjh;QX0|RLxst zYL^(Fz^hrMx5sC{ldeI{uBerTtG?MM2YEz|G;dbJMxGTvJYBPq16q9q==yGhQYG z{b%)X-}~37i*D7k+E=x`nxwu|i>o`7hUy>nqB>M?IeT>cPx!E^lB zg=WT)m<(s*J!;IVJ(x#!_sz1{mFpn#d(Lz=YBbv-Cx{PCw(^)74CbuV^It>u#f;PlI+zkZ$h^r~5d( z+kf%H%{Z7mNc$&3C#%iWz<*n#XRA zGdPSZZ3o`R{YYh6ppD#R3c~L=8qb{rOgTm)g_(C~EbNJ1!?|eqTZl|mUQ>X>K09yE)|jbcnZtG<$aOekJ!K3s<-q2tFgEwRsN%S>W=VAu5!li?xeHBuF?%yMkBYQrJ12iJFfqTsZc2CWWayo z>RqQh^d`tSIdmQAy--}syYX2*jf8555pUSxp}vKwW)>Q6&tP9W#+U}>t2;LOm65Y+ zj2^jUJUtKL&&<^iqfd4uRLcwS)THBYzOEND{_4Gr{>C=6loGfLXP~Kc6IjNr(9?Mg zOvi;xL(B>Tp`K621hoKZtO>L|ErQ+dGia#!=pF1rSHgj}jBUs6#U1(!U5!*DgHD_W z$O#l<>#=`9L)yr?k)3VK_M=1D8+0#J?6r9P@0k45#yr41Gje%L*XBd{DEY3-Fls54_gd(OP>~FQ&6_5-!ANq%ldvmD~qk ztDil?t+3p(u5?}yx@cnvn-{5xs&qOiP+yfo z>i%G@D13G0b31HQmD|6hMI-=g5UK$h4*JR&Oc zFeO@@jg2VuYjigDMy|C5+lV{P#aoskr@M=PY&pOe7LE%Z-N;)n3s#%00u)haZe65$A%->R`Mr`^<1}SX1X<>n~w6 zGV>YO@ummo25Y1b^wvqgmzFIp^>5{rgQ=tbPW_#beE#o;l*o*i-t??Bf%ak%r64}N z;mlnwzfje2&2ikF&3P?khjYF=!x7;g=c;LSGkdra+Fik&=#sYn z@BP2u)2pQRNSow2n|a+s{bkXny;5zU1+;OBg#W&0QBx3J0KQhVu2R+EgE>5tMorj|?1ks6+s_fO{Ut-l(h zt?9aFcg7-QwVl#?O*A~OjA5fw&O`Q1#`_evd@f0%a}#z$ltZEv%hi< zajuCd7uGPcY|Pcz%(i9%sTF?jNiO(jRfd)p4QJn#F4c`^1=wq5*GZV&D;n?Nfed>q_GltVg}y^Id5eR#4MTo>A& zT?r!kduvTNf)Cj%hVBe?hO`W?5&brMzPQeD6QX8?PPewO_C+G9Ip2&uXdIC01#3PyFWc<#`@=pQhqMwv3KT(Lj%Ipt{Nday#@8xUp$?OvD zq|l6C3@__YdJ?Y2W?*w%RYIkHQmlA0P|)`?tD3)OKng7KEs!3H;qpCch2}D@>gCOr z0!K(FylOh5hb49|h4>Hzp{d6;Bckz37G zqS=x6$b8eQ$*U?`pP(&Op^${0IineFVt0f z6}04nih^CwI*_#L(Go06^Rln;sz1@u;E6v0L2MOVDK@4bI+y#vK~e>?=VY92qi6$o zp(FUQm;?`jEW65f&%Vep9(Rc!NWH8JA09C~CVO;rtQH*;_uJjddB=U{+JQ-63Nf8MWHNX zr!JgY>&t_X|DLRjk?YDk#5l3GKR)o=9~PLCmEzxw1YcYKLr_}|iU)$vq%?URdV>DS z&y)ylnQ~l9QVSZRlz+8@$}DxVRtimf*Oe4dlMjiR@`-uvC4D#8uHVzI5#zf zX7zL2Nnb^)Z+<#4#wgbvcSH9syvoIqLqdy1?w@9rMuPG-m2TG*p4rOZgE z@d11l1<^U50(QtYxwU#2v;QOCJ+XHn!?)i%H&{5>$M@Xlmad}LYNmJw^baMlKslgH zleSBf)#~~PrMy&0DWESvwwx4BS$SKUHMjM+J)i4e=VF`{f}wdrR)xKEKXx~_U$KvN*S6)bbmU!DnY5)n z$Pao{zlp?YoN+_y1ihxY+((=lC?h6Fv%E$8N&e^l%UQeqZJ_jJX5I8Z7Uu_Bz{6Zk zE`@E0Ay1Tf`Hvi@b(EJVhvlW}MR}BRM4BUu$_!;R9N@Xs&1!!Ai&o8u#_Ti=N?--- z8rNZx{f6XXF`RGXp*4J>?YZ38`SzgKao$O#&)MqSB=#0-<7T5@V>0)fE6Dxj8uGDx zZ>}BRMA(Vb#Bj@e_;-`x<_mR}vF&$mcC~d6!c#84>ze(ty@h=f&N7pP2G-p~;9G+t z+1<$0zp-6GQ1}eDcPn&f+z!5j>gkla_-6;B1Bd+meE$U=2jhbv4oj>3W993CVQAhc zq0Cd<@>VTc8m$(Qoak*T4_3((<%U{bS)j~N3nSq_T^VZ()_-G?T#21m8BCa+Ft;3p zzO4f~z`pAxy#xlwE9{5g zvQffCZi?j+NBIV}i^4(6L7}nzr)96Lm$fTW*VXKYh5gq4wk%6AzMJJO-<*r1SGlj? zT^%J^#xf?s$geHZ+TngOK&&dq%LCDYcrS2PdgiMrKJZtOt_4a3cLhFx!jd39M2A<5 z+){2L8tPPOI-I9%)sgZ{^@dbi4O0gzZQ#k=skMa4u@t_MB)By$fcgFzJi<040Z;n- zn7n4N$4ECe8#e5ZXf*eOn}{rT9qs}9m%CsJLPtCu8wt+)|vP$AZ~3 zoFBuy<9}doe=975r!I=CVrh@e^LoKy$Q>cmy&lDHr{gT2r6 zh<%M5(5Zl&-3SrQGsW$);G)*_0`FQyxmkl?rmE zvQZwbK1FL&E;UUqq_t2B8)xMvn%!`tL8y?{(|D?FF~iYK)Yg2bTa86}J+_LTsZ;t5 ziODu*HChT?k@t+;>{@2IuojG&{d@wy&q!qNTV|5x982HPV*GNZiZBpvqYJ`tZk_oA z$#jBuXdE-p>}q)eigF(K;%<>F+Mh3l_J_aNSe7)O^IymSJ^`D-8*~k=&-P@Bl4{Vw zQshi)PcUL`VDlRyq^n)&Oh#8Eb0+I5A}A8OD(R{Gj=e2B!`(!T0`O80G{3yErM*~7&#lYUTozHC%J@vHCB|^^Su_Vo zkR|!m<}Ep|W2{udjA6Q?2PLOk()LKJWUWa4=+pW02G5P<(yXgw#oC$cZuGRaB^%JT z{Xx*_Qu8U-k1GdHjhpR@F1>S10rZFz)#nQ9;MJ5dYnRpb@+-|6S||CbZH6`(#Oh}1 zdRd@jNgi~dpEA~(iApi%fSQdt4h{Q>LDh<8JAaxmk0xqS#0q`mE!k~0ho9~YDW~ zbCl)_?%_G8POqfpwiD8QGiVtAZc_ofpydm8=Q3?8GCykdLQm*2c_u5k(tGGhOoDvh zlE9SDN_Kvb%9|CqW5H9l=}Ig3BE05(%Q>kGk}NSyFQbG!Ls&toNWa;x{CjOBc%dha z>D&{25J?mbVJW*_EN5NMKVrgh|CwdmYqDAb)5tMcG|-ID%Iv}A4Sr{eb48d~IT7jC z%_OhXi!E%#DqXn(@(@;+gU(Iwi#_RD>_3CvpktbK)) znrKNS>)?$$sEp#rm=Bd3d>fiam9^7kI6F$O$=4w@nD&)?lA8#c_*&>< zJ`E0YodB(*pHi6br=LM@b`_%?jS|Z{TPh`JRaG?_T5`x|%{!o#4N|u;CjI0ez!##) zQY`u9%%0#~##o)ze1vOyc8ujKCQZG^#cQwA+Ww3Mpx$+VU_idJJ6sXf^D$_ath zr*cOaR!?L(qYl5A{ZI89IdNV}HM?8KE2g%RK38_R&Zw!zOI+iBEM3H@hRj}uey3_V z?PEnwn_xMvPSIWn`9y{OB_4@d-)a@ijnY48EUBq0*snW`Ad^PjaudeI-%%H7X_otP zxHj2&Nxr16rJVMMjrAonL%4b73ALBr7`wD_+AOZ2cFpKcYb$%nSvnWzMGMtHhDkPW zu`{*ZS~^{8lmyA2Wov^|TguFCHdIGje*_|h75W`G7yn`M3m4=Tq!m}&D5upYsy58t zUa3R3N*jdL!g%G7G0oCZ>2DrjTVjviz_-=;o+QzIffW9Q?iN1ECAl_aH&ak7X6wQy zD19{#&80S@hq%jHQDeD~thORW_!>~a7ph6_C;mt|DBNOH{QwJ+h!McM`BR)MlvO@3 zqy39QOpQ0Z{2q_WMS zsH^oNbPnHIy{Hs*MoS``zUA34d9AZ(kkA|EHfFj01?1#(xsGL$S%&?iUV~$yntq-y zqfg)yK^||*BqIH9GLkY9ZL;H(F6~5A++)wu7VsdlLYQ$vjo}KK2ianB0V300d~MaH58|(qUHUtu9JVPr z_12bL+7GzKFA0^=&r}_5+rs<_iL=IvFPQdZu^z#UM<;nF^&hhl?GMk^Poq7$3T2{zfRJqN; zYiu+}vb-IWP0mfbb2-d5%uCz@*3qJj6P}Blq`$ImQCt-P$xPux9N0kEzWCAw0$OJLi9Cs0{O_z5qY|w@#@P= zc>PFG_O?+QDUA7MRlTHjk$TLq@XO6+Y8z4wWQ@bwDK4C;fewQi+8(qbX!MAAlD%RU zQpb|hbh~kaX-ndmXJ)#>3i*(m$geBNUqqXW(+r`5^s4MuBMyF)omw@r zhza5uc-kyR2f}|^5Ki7lOe3Wyzuq{&K_ptg zvt{%G^prLh+WT$l2VXJEY=vx1X)d?PLdDK&wl@jW1TEe@;0+l=dTASw_kN~rMTW2$ zb|N#(u~6l&lkafT>{kD!$3Rsc4$qTL-ZHbz7T9lefQPUu*<|KovUE{j$aJLrLAdhb z>3Wu#tl!ZM<0M^&t=vT@#7_MPvI?>0Dq@F9b;xK*#%nV$n_eaJ(GYc0o2VDSY!HM_ zmXFLg#*>oBhm4_T@f^HkUclA5U$>&gN5P(CEs4;}fS1%1e!!~eFFZvS69c~aHE2qy z1=jQ@Fqb!=BmR}S2U=7nQZ(B^;LHWDQ#e9GFePyR+JhwC1+xz_(bY&d;}$c4$_>vnQq{Z#m(*DE6Nqy<=mX&(4-~=cWXKrvsD1}&&nj>bO*e*`hS?V$5)*lbp`;=9 zzozj8|LQAb(ca@kDuJO@2%E)fkCRec|mGtj~Zq?=#Z`9=8qT9$jb_<+6~^QgH6IAZIm{ znF4O%W%L9Nq?^b(nu~T|n}Dj9S18O?w15+1>uRg%*y7CT;$8pqdGrfq!z)JiiR=~O zjV|Hb#N{#93Muw8Tn+6s_=rUUkG*qzyFITn`=!0f%Hj2TD*WmCSNGlU_sB}}eefL& zp7ys?{?-2={nDB#k1UXro6IkS4`z(*h=t`@cB6TS3hH6@fBvpMmv55)g46~q;-la>H8ih-dd2kO}>lPTKynf{WRmCC!cp;nm?;9n0>d> z8>Qy-wfFANtRnXmBekdEV&jNnN0*aBSk6u4r_-*&e|&lPu7;yy^CNxFlruxs7Wzc# zj+Q9(4nFXX^u6_M^0dqx=Se~L`2w#in2@zB&`aE^N0>Rx;mlRAQPJcj%t7vT0Kb^K zz*pmTz*%&gj%0gMiLoP7FdCllRcs{mg`9kUI1wLN?g>3@pIqNvRYQI{vV|9q5Mm?a zuf@jal;SVPWJdhro4Ceu6Kx&Ya{3DGacW$!a^`7IInV4g$v->eYbKkuFg*+&%Y#N| zF;r=b&DS+`DpC-m*#b0;K4hmOJDJFC$K5}VX`|zfX4-OfIT}1)sj0y!3h$pO{>>Wb zDVrJPTN0d`MSMra6Iru@Y0`YdtHz;^-N9{R>f-eG3aVC3E{W+!KtgBEA{B9+ZORlf z4w_eJZE%nVa}ubzorPW23)XYOJz+HHiVI!2-J@Km-L1o>Mf8bonQe6TJMq19G>Bdq zv&cqm4cLYDPP}Yh694!o`~LC0NWYo+J2N?JqPL*yJ zE_l#Y#n&!aA?s+Mch;r!HOhQ&r51Y|`&!DX)QIF2Z_1UFFm;wO2cF(&_|_dza<&`0 z<=N1RUK*cuCwf#5il4RJid*Y1^$^?3im}{C zJ`H-;7C8Mz5T3EX<5dh>`IXEyoVOO>JROD|bO-QVu42;Pi&Vr5a*nCa-sJm$SsjW7 z_z~9RsEvE&u zq<;gOq&k5fBJbZS#Y=rbrgVeD*j(HWALbwInS0_pcv1fi!t^I|8CqrAsE5o>#!mQ^ zVwH+|CneH9SN;rjYn5mfr~3Cx&(QvUJkV1Ll{L+-H&X{80Z|No#TY#KDVfMbuq)|I z^i(Ryqd$OSD-*1*5lC&9!Aa&md;(|82}qkVNXf1xIp6{>Ed1q$3j^T1?_up>yXBYy=>!>&p%%~g6))X@-m5{;eL&ZYgeBksY-r078Zl?SI}$(&sZ~Bi8O!`RhzrUO7QTX zMbh>gu2;c4u5W}F=oq{Y*`OhTjEh~+Q#hIuq5HMKso^uYg1xx2LO6IR-E4<#qnvvj zO`Nw}vMVYq%H1~VWrQzcPWYS97PiYyA1gVJaC^DZI6GEFpW<@uALWEJi&T=2si%VL z#D!2-N6Op8;o>0i8kzzIi{YYE=D_;utR_pF!OVE2bW_rl0qPL7u)J9Q3DQG7#R4+! zeZ8hKP0kc2$n~_^Dpm7K9<`8~UmKzq*8JK4a@Y)lOZfzDqg41xx50TFic?oR_-m5Q zCdi9W@VM(Cjn@XglqB;hscpVQg0?2^KVCSt4%0I<$nIhb@@Svo^9v;`6>PG#zI}sj zts~4a(^OkXCfPhG!5Nkdu-|LylGu(HO;ZKBnY~V@M=%$lkpid@d^iu zg`~0StKcZR?WFt{+#9PR8r@vuLyv_kqUD`H8}2ad@z{h;zlo&*=}XJf9~Tnm$3 z#zRoFlT^RlTpI+!<8}CN8>w)(={>Zf*dSEb-x>*K4rrHQAO5+E+1W4d`=lAkz_TfF4ZcTv%z5=r{;y9WH4y)6X7takF#+nc-r=YB;dw$c$fT#-mbN1 zhB=14!bH51i}XCR5E+vr=s9T3jD_!|Bs?!QNGmwA?i~-h5MAGmFnsPq@fmWRJ7q z{2OwZ>@doZKk!2d`WN|=c1ZcEWK%wXyF5vJEX@>02QB^v!5+$5ac@8g^abtUGMM_c z<*#a2<)|jAEwoOW(;R4A*W-~@tbp`eHxQ{`=!-B-tv3hhCD93W3+cclJTYgPAMsl9 zl38$?pTIPZc?{3X!^l2fL;B(xnl6qRQ;_SnF-4IcEN>11H)IIyjlAq4@|;1n?47b9J;tQ}pj7a_!XmGvQtI*+(adVg?OC|Cil(=2od#)UxlkLtewd@f7 z<8-d3P(&!n_7&c8o3WRA!PW;kEC=0zUF14UA*N9RiK@;$q-`_`n@7!Upk)*#JD6;wkg*B#*#=S> zQ|BxY4jw=a{tk!9W?dy;;7-eCWG7qELXrTVQ9Wk8Mwnd4xy6uNYET%bCXhUOLuB?Y zkm}lPCYn!V9)K5jTYRmP;lF)yvC>W)Y(cTMf*}<|IWs zz|DqhaU?0kY-P6~iM@?UBO1FxF9yQ#cg)6iB!q%=GUr3T(kA8{&4JI^R#FiANFB2T zp=FQ~jzPX|yRll&i8N@ER!(hggfX6Y-BOW-eQL~idkU(Qd6;`-(+?*%c5^7O}|On>UGfD)d=3S2hai9(LqQD zzBDq>Ls=B0&^zi2_yMzk_shlRHDsZC8->vuSQ!)W6M7zV<}7w7(?mPR-!)(8-E4KtetJiKn=%q?&$5K- zZ`smhfVqXcj?U>$%m{r46}73%Eqw{}tW%^7diVOXyEU7kn#1WN&4NwBWafp&a_fw( z%1-_@UTJZ7#Mhe|*hIO|$5LBAh@P1=azs6<$I=`sQZl4D`2Ww?LFNwmGWz>Q!IOU# zb9iIMG!l`r%=jNWf<~$X>2s!?zMIXil{aSSGvMnD6K^tu;d+aJ(wYg)JP%h1q~)jJ zg04UU!HeAdTK=6JOP`W`%oeahhx2`KSFLR+Z7TY8>esH2>Ac9yHCJc>J92-dGBaO8 zR$p6aiL(rl7n6tRLhDQK8E>>4!YK4m`t?L6o-40BdJ5!}ENKcK!W37lur1+N{A@0! z3<$S>Nh0#*9cekbSu0C&8{_E?bs_%L4aRCFk^9PfG#e)AMRB^kXH?eP8!N%)zKN{& zC#@*kkC|##P-FSJ>LKF>f5a$=PPGTp5TOKfUOU95%8$63WWQO@_z1P(o+b$A!AQtU zD;NT|0)Kl=cCqFlxy&r~Ca$Exx{8$PCA9=wTkXocfV$#uhXrO_) zx1k9vyGwN7cU@HMPprZ5||yU;djkQuhV@@{P=I)5tJ8>3lt1{+W+Ge7As zy^yLf4BtwfZf>yVQlhX0>uqc`3!2Ngi*jpoKee*ev=$6BPTUDjfZZ}utA~EOuSR#a zyObTS*OyT22Z6k?TY1c8&|k`BwkngQh2tx0u6u9?NQ0(U9SYPe<{4K9+Ur^9bVtZ? ztr~jCEHoe4uQp{3dRsrNePr*G6jNqTfo#=}79pkeLc$ZJ294BD(sJf`X0_IVSz})0 zze7E|Ef+FpP&et#R##i%Gm~sg(7Wr)*u3UF_8`~^H}w|Sms9|C1PLi{UO$42a2wrv zWwkQgKIV^Z$3AMhQj$xCi`Ao+=9VJazZ&kz-Xt7kUne}Exz$T(1=z+7VCJx3Jg9Zi zT;%~9<286`o#6Q{PBtiW%{KaZoCMApHyPfztVnbljROOq6a2*A^!r>zkhmWqwY$?8 zj6CRiqa3M=Hp?gQ1aC7#X(z+Wwm`=3n%N$cO9AaAvlUHJt3lsvu07H7vT+Qjt)QDg zRA|7RV|3iR{>8MuhW!oaPDN;{26G=?)}?wf_}a-Nj65MPm@@hmas^MK&PH2W6sPK& z*pJUOS@Sw{!fotSl8mddH9F%s<}6#08KbvCXW1zbIU|j`^b_07jAg4C&B1xd%e;qv zGy`;^o;u5w1ufl0TAPE(AiV_hS@Rjgp{aY!2D(izWRzy=!EHQFN8%C=i!JaWf7C(v zA-BL2fiu8J04rcV?zY9rZ*o#g)8En7$VZeku7E_!>IwRBCX+b=7T`6y8LEqeY-MVY z-AJaZ?02v<=WD&`C3H*_HF}^&X(ZIaEnv*XF;9(HG7Lc!4!&1PN}(}uA^X96Kqf&6 zUP${fImuEuw>Od+aIg15FHA$~BaM;C%11`P7d{5Re~NjL)PUnYI}>g?F!NV4|1$gF z;QfHT?|tonxkbNfxb?fp^`1m#`L8wwPW!)FFXI(Rs+BN-HHF6dP0y#tnL(r#a+pi8 zANr#oHkaug@NBN34>Ok|HS0rJatE)K;V+Emj9N2f%l5PwzxRhCYU2Q z77YxWRsD+o4UD-sOew?EL5is^mupFUpa629 zm89NMJ^6(6Qd$-qC*RO!st=43@VlM>zas&@uvuWX%;(GUzqu%6pw@Hch1^0j6QqScN*))wa`)`p*8YlH~dg!zNmIFJ!3Y;K8s!zCfT|eC7Ax&Gty0L z6Da0w=E<3UJT>)KtJD_1qcd81?;=&mcqe;ni_g>|$X|Sf=EShWt&>3@ufezC);U*L zCRk6Jt?4b?x0V{`&1`yAr6<_=w%`n3r0PGv1u9kHmbo(kO|>l3JMSo9ILIC)!JM1C==s_(u!50C&<-9rUk& zbBap^Q`G9d?=BcRH{@yft?1>EzoYuZe2v)}S0Sc!wx^Lj!@CP1!7_o0p2Pk^spUO2 z(#|AHe-0q;-%BZ2|RQ zoIg5P(sv`+N_s7Jk*mVlX3^(sh1E;yVo;F2nQuTB$*Ux47PBfEy-Lb~;C!XE8mByx zSaAnbm3hdTUsIlHz13TWr0)PDXc6hoPU42J!)O`q5t1**p`5H`v++au5zrSLY$$)! zatUo3<3Z|d>B#4@xYva~i%5w1H|BS?L%BZXUX|xs?)N#1W*Z&qQ1W>!e|?^>f6o8? zkzD52*Ke1;@BTg_IVPs)+pBV|o``K)KkM*f96Z+yl0`f)Qui`zFjP0j zvkv$k`SSX<1w(_4rSnR-S_kx$R9Xk5fVb$Usb(e^m56Q@VV;;%z;npZ3z)aC{TU5T zK`G(`VW$IlXn%~`WFGsTYb&G+f^D|_t@Cy$6WS=`V~7yGGNOLOm8ed!_o83K9?bqb z{!_w#Ij&}>u6leuZ;kiKk0-u#y^ML^`FDQLeE$wPS*fQ7*tM3Bi0j#h=E#$yUF1v0 zdM1~;$onv*aBBOs3u&3@o4xBiO4_#c+}_jT9X*k2XqshC z)pklpgO$AT>7JCYDZ#WknK8cI{ycJDeLeYyhJ&G84Ew**%ncKJFSV-FR@@kb_feS% zw$@CXc^ zBOb^_KjygC?CRm$u4fOw=1Xm^*sTp6S3(Mh6?7L4n-y9zyi&xlm?Zaj-l_~u=aSxh zJo4ppQog^tG9BVUeH4Agz7nb-LvYmA(p@t=7`7|4l`B)IKrYEbaBYS!^>T_ct!74J zUwvt{`bS$03gvrp&6vZ4;0(B(wq>4@Q^c%SJ*@MG4LHl`#ScEx<6}Ya<8w6AK!iXC#jgn6Sz)9s{n3&hOLg{vi-bm zn%xcW;6K7jVKH0FP^1&ym+3|Sj!t=**3o-gdapgBwLl^631{JWEM;aqbMA9C0bCPTb{$%>@<| zaTQxxm`!+W-;iQ|kagzk(IHogJ(>5jzVgx0G&UpqzMQk-?nh^a$Ga2lPpzWmw%Ja} z$~>IB==-hD&%TxYk(eF`$ofpajkOOpa3@^f(J;`=_1dX8Z#m{$(Y*s-=Ob@;+J;n5 z+Q-bHf$HjbV44{f);9qqt=P(nuOuhA?dt6lU-`Uj&My#RVpAK|W0PKe^0gDJ#< zxKM)4%e)3ztuvWOpK>K_zg*|T7DQH#^~RUW6`uE0fg**r73o~Wp1)P}0nvQ3{M5SL z;$ik&qvsdWs_~U#Z22x03Fe=dCvUa~Ayupn{sTA&#dJroq-WC4@}KLxpY(3exAT9t zidXm(uEg-7QB7kXN1hDNba${X7X1HXDv^c;D{u40rG5DQ;%|G;Y+oa(xwhT(b3JS; z-DBL*t_!vV?v)t>es_CsO4`@-lb#cqrvpK`kFkQT=Qw^ln~a3zQ{)Q^(E^~1;ZuX2 zv2%EK=Y@u=s*8;(X!Pw$dcmVv59!C*WD0G{?cf(!i(1=Qy9trpH>RO}1uFj>@aleX zPi^&_dqVPuCWePZbdPKteK7W8_S*@oa^K6ppvc0)4f73-2F2;C_iD(o5(lPS$au8u z*D!6Xt8T*Ggm$?f#pjPL6M5N{*WTAsfL~x75YKsf{W|nv?z{Y7mi#D_v0rV%<*~1H z1RzH4dR_~Qn5;%#GGQXQb}8`*^r9;ft+0(oX-a%?KcVONe|i~H;c|-&A_mr zU*4);LSpVL($?L$RCXT|W{%h2YT1yVtc-NfDmsPPWPa8Mt54C|nxrf<*3;>hdd@E) zwZiL!tqDEjZt0xu91)^N9FDph(>t1t=^RTF^gI=F7s*j0Vg^6_&$%b}E}uWw;9ldG zUH`J$Fl$Ws`^X;I`bPbYXc4tDv{cAM*DS|U^HlJ1%E>PU-;RA>C27dtzTz5E%#sG) z$6Y+rtJwzH&pKRT%R{ESFSt)Qy0iJs+De8$dq)3^wHcnYhu)F?jKCWyMjJvZk%nxd zMHLoXrVCBE1t!{frFCK@@v}J0%LH~yF0laoRzqkV6TN7B7vnBj0XkVgZwInKA*4NI zbZSor9jF4SMRw3pD`!simzIlZO^jdU zE0ULXzN}ChURV|?!9HC8ZEdlBQyCGs>ur}gKQL8!t!B|!CV`YNUYPg^CSpUV@gOST zY4Jh%p|V;*c#(6OW6dMT$~HFtnrF50*e8Am>H7g2&E_FL;aZXOGw`}hr%Bvd%NO)- zF5*-9*_Q0qxAq0DPa$K&vLd%f#YH>~Z5z_f(Z@MDltv^+mCyDmzEE8LNX{;>PH+9> z>Yr|ZvivOcdu`SgYz{la_nuuDjvo3y_E(OY5vk$l-IFa_SerV~x5*oxxhrdE*1EvV zK*K=QK*eAs#i9{&9-rIcag1}MxnxJsW;MgX;+OLMtgmV)=e{2tJ+9MeM}oN1$6P?3 zBf-=M8Kw&S4s0qOSQk1PM^cU$XJ+(YN1Ks3r>K)n-^WrgUK35q_alSma zfQ*Ft;n#<%-=te&{=hq*f>-`S83i}gL6T1EgLv=`O~Sd*#B+&ROpY3v>U-^mni*U# zo|b<5>w%+ER+gkQl3S^R?BW^CgKYgFEeG}jnMBtvGqa8H`gVP+&Y{ovBH3^3G_IKa z;FSHWVF#g?)+%Y;!LjiwhqZuG4^)#j>Q%iFNG1i$FiaP9*!FyPehoJO`Gaw6f9#*9 zVOvxQyM%{mm-U)$nQQbmcUkaRD%uilW37d3QTBl2iJfzmabC1FwEl-qo)qhN$27-d z$58io*C+Rk5HU10bZ1Ca_X%5G;UiO7Zzp#RH20Ih3{SK4u30BD{_$+`ZId1b=Ycac z9&V`i+frwZasrAlUUi}GrABI2`%7^U7Hjthk zV~$n2gGMdrQGLMsyi3x^7BHO- zqeto!nGA190yHfvs2M)&hTUl0y2;vv*Fp_&JEG9wPzWuYnP^VB4~lP`#ls)qY71+G zKd=FG0F8Q(eV(nZb-tybWhI|ty=Loa<&ZW^F;lTkKZd0H9&*O0pgj?Hi%X@8>MryW zHq$yHFLoN8=Q*+GjYXT}Po3(l(Mx}zRl*aYomEQ7HXyx_ygxTi2St`!<-!dMU4i@&2O^99d@ z+Mp;k0MSD;4x`JU4*g>8C4J%5Yij00b9_UyD0JDMV9zbY=lUt-nTxam*gQv(lB@(; z;XeH7zeW!dLVFm)nS#)Zr$Z|%hpF>lV}$-28ICM$*Z!K7K$c!ini*s8H-0cj>qqr! zW&=<<&*QEdu5ZI;V;^+p>PWlnG|rGZ$nE!J60u+Tj%J_`Ge4Bz2^BF{{;cEiJWn3UD(g9bJFe{EApdUexhqX(=d7^-XF zhn&q^Gfo?|=tgMkd&mp@DRa!IN?7zAOgCnk8)<%Mhn{{j$okFqdMDaNw}Iew1Pt;d zy*M`OaM5dpxICag6eZC#&KzdchSzpEX`)SGKEnOcR(HTHf0e$_{9wbs*0$4RJtuQX zAC7%eHFFmonhXfP5o9HH8cV^k-Hz|!H=HKZ85^0UPY0v4t;*BMcuhNuT+Atu ztJ3rj*!btFOUhpDK_+yJI$d` z`g4=V__V3mmwnOuvkMFYo%f4oW#KoGb3p4YL?)T#ajLz6S6YW&(u#v3Y{zq@J()n) zq8Gj<+(})~+uem9r+?IP&`(hB4e*nrKnX)TH_{uUNCC9?Oro{nDi=T;9Rf{&uVH`=hJ z^hrcv8yKO?2s~v}tuQmdG6rW}gYMDKayj&^%oe>QHH@~fjUWqce z2mdm2!fzV}`gcF|JhN4Q0B#C6tmaLSh)*N?^a&*M5yoQr-MoROp*F_9YCAU7Y_C@3 z_JPaV4WHddq?y8VO_&W#8+{ntLiOlB>0eq{-2<9x52k}Yi*2C)CgqGoekRF*jVH2E z>=LpKeCF=vTz(XIKMjyRYRB$VubHE98Xd*HRL;XC19k%3;)TdLr4h4)Kcd&7g^V2N zS6xI7V0RgAKBayskqqHNNkRRGF%R$JE_z?Pf%C~lpJE33LUTGdHQDg@@`L?tof~d&PK^pL7yBTf_8%$}!syu+dM^ zsbC@hA4g{a9YwY`@%A3e#7RPMm*DR1?ry=|c`O>-b#X#)g1fuRvbbxogb>e2d*A+Q zzq4o0+w5Clrl;rjz5lBERRgOEv%#>NU)a(}Z^ZGYH9$VeFJdm69gHY;C?%Or%g<$# zxy({aRC}@k{C09!Th8re0<|lAL$VefusuQItO06vQ+k5l2-?g2b$u!sR zqCajlHwk*AqF;ylTann{IbW_8VDHoAt;u?Cv7y<^oFUAiKaxz8$yU(s(F?h^8c!~> z^{GZAk@?21HUh2AG$CowDQD@TaFev60;$#HBQoHP3Gx`s0KK|-feL1Z;%c5l6KjXSW2%9Pu+Wdzw(hfz~z{G zS$tcpbKr;5z9jA`{NW*b1NblQur2k=OdTuNxQ4tq%ffSG71Szlho}bBO``-T$(z*W zTns(V;*7mwuF;9f)N0f1kWTMvR0OB&t-m!B21j9jtP3xYqWh-S0G&hv)o!14S!maR zdA?eF9x2GNU;D@eBM?AFn+-PWRTv<-9!D0sw1ZeJFqL;B6Ki*S@*g8 zRt2*?H&Kf*>|zQtL~Cow%oTka<02W{FS)yL(0ai8Ed~Vrt;FIgs5gv0!f30j5l_Eo zSCXaHMyQ>A@O(}L&q*}Ud(PG|KGXB5i|SeIBf8TeN{+1vX!N^|wI<(~d;V7(b}vRWze$_VwUYhLtHw5c7n@EN zQ#08RBSw2X``359x=LE zNw{GwrwbavJOZCqC|_AxpanY?OezA?rRF+o~#V*Jmexxpe&h&}6j7#9`#hOdWLGG%*p)<~(O*ayw;k}9G`s@4j zV%9L5SBfL6*e}{&R1i~MNg#_r7MgE;l3n&$N@-^gt&W)t#eEOwRHvDBggWq6M6l8P zTFbHquvh5-t^nz;cA#o-k3r6AU^W(hkV?d^(aXqeri8zfEr{FdAxuR+lblmIb3N9*xBeC( z@AT#RBkUOi>3yaHiIDH~OQDSNS?JF7L;9qpniN=HPBqTyubdCqo8CNZCB8A{dVARs z)<3#~-^U$ORyw=-3ZeU`CpXYfQM&7uxm3Ey_ve4oZuPs_kiTwrWhs9P=Y4WOU(3Is z;*GcFO>qxq?hlaW*u_pWvKdhMc$YGiPvI)7SVyrI{}KGwO@C)5-5f+8(vpZwCu$Sf zjm*EwUe~nj@{YmQA9K4nPdX_KHg}q}YzzGz`MYLOYMQOLQr&(=c}pK*?pRIe5&BB) zHl4yN<^y$w@XVA!gnL24v69yzwVA8J2x+H2-uVwxLRn49Fhgy$wo0mIpQx?|sjsRr zSj}~`!l~lBuuq$51qj7KsaLJ4&fyY87k1@S6WO2CUhMBiI%Y`k%wFOmIvP8?XUsLd zyBLK$;a@^ta=^MoGN}CQuguwDyM1rL$Zlsn71NY9flqU{@pqVkd~bicFw=~*biw9p zFFdAq@(;Dc^jUtknIBWK_i$k8+AwD*RZAL9>f2RzrvJ751u4rH!@lA(^|wgmUCKl^ zb}D3WPUjFEtp;Ds$>MqcJ#yPN&|0U*sExP<>|bCH=hb&m@A#ccnp2}Y_$ok0d&e%O z-4Mog^R(rzljv?4awEmKE+ceWg?Wd$6iYWQN=jqt_TNNZ9{G|cB(3-F^i;FA>D ziJ~L-yWwOA{i#OAR3YAq^MyNPD@PyU+a)9` z@j~5zU$V!z$u|=ZW=7jzbJIPnbuv%4tm|A?roVMWsxPdes%jlXlisKV(owdKa$PW_ z&&YSoMDX{oFyYc=?hjuW*_?FhhtXMM+`IJl$~>+K8|CLntUE;KwJpeZ9}yFBt~j3= zKd4IdH987>^BwYgCS8AMn`+%R7Ksk#sDGr`fn+Mj9cQ(_lxv{md^Cr^>G4&_RJSte z>JlzRc%f7v9o12;(t2~FEmKzRV}B?Oa{uuoq@rDg(`gtO_E&Sy+0TGq`7b|7El7%q z4~d(lykl)g^*yc=%D?Iqt|dr;r47T@+MCSolST!_Xrrx@LPcL7%d+##@^l&M4Aaja zB#v>`^4(ENI?ovr>27}|b+Z+cPnkU(zpQ@heCm^}r0Jnrn)l$|?#NE0qt$L~b#0%{ zbL+W3dW_{Y9@)N|l(dB6#HQ9gy1ZG^oCY_|6?P`GO)mlmRIZpv5A}aR?yQeh(l%a; zSBp@a&}gvNxm3$B9_82^3G4~;r239p?2I*9Bgt1!d!^a=llZ2ekgn+mh4JD6ZJV^4 z9u!pDKLk#kw)!Q#o>12&`|H!D`9(az#8MZOv2=oSF|B(4#|DaqG{;DAp0iqchpB;r z>MpBO`Uc~nqbixLBT=FRGG~-b#{yTRTtU7m3hX&b@a^Rfli6<7SHoCg>p&t*o*_IY z2ue92!Jft}_g>CWw@&uIdhogO>;(RndV=}XqJpeEJa%(gaRcZcI zejdA3+oa#5-iWkjNR^oe_U=qsWhiMajIe$f=fRxb&A-wYnz4e%l1QZTLKwo;A_3BH z_L(qG)u=iA3Z;XY%TD93lfv3qQs0`*&*w`aJHANUVqLV^m>Ec;pQ4M{F6)Gu!yK14 zvi*#D+*wkF`hr%rI9DC3wJ*PN$bLxpqs?Fx7)^atCEcG?Lfe$x`RKnJPst!#%S%{Hd(G$qOTr4%gk_n|ewfSjE zor70U5~&Q6KY(5DI-Gw!JVh4a`CO2y0^Uwls=TnvdP%n>XX$n5QyO5-LB5Kn+L?3M zRrGm%0AbmKHbZ>_B4i_>zvd&G;4X|Znp-y<)#z|TH#ue(JV<%a>hgt&ux1*WRxM$x zF~rPIm*UZ#Xg$z!Y)8$4RzH%)MB$z7M0;jAIO0b$4(2hHteqm`(W`z$Db5|WijoIL zx>bR1fV1cb5EQS1X0VW4W9DIR`v>#Qc(x_AUasZ}gyOpztrlCUiBL@T^De5Q`O#_w z7MeycR@N|adN7S!0D!h?~Zrq6+AD&DvBrM_4T-h#$(+Brv62%zdwU6K(p|=y+b35ow`o?ahEWcOhOKmL)YkRy{>h|dcxc$Z@DsN zb?6^k1j9Us`BerxiW|t*!OW%<__j$*5vCqrA5)}KOndPS_Y&z@zc7JcL$^jc@(kaG zjm31l3)h2Q%*FFds2s*kH{>sY@$@&yu@u{lNw%I-N0BSeZ#1#8jn-(Ynrv(%h1811 z9(AjJL~Cr!QlF}`^z+J9OrJ++oVr$RiiXAB3a3BN%9y1E$jFc?8CAhkew-GJ$cq1U8$f>>??^`R{?{ms#I(2tHVT7ppd229jWnDkv{ z$Dy}n0R4lDr}EecyP5yPB=SUT&E$8y5E9w7;L0oFLE$^JKfAq%{hiQT$ZuP1Yb>s? z$2zMDrTND+LU+tBK7l?%>JqdyncK`m+B)d1YxT}rDfy?rt?ZM+m2}@DX$>+Whor_x zDv3%}t*`!5uWck5C6LdpL*9TU`-j`lG-eyp|Kpak4sH+Uqbh)D+!6Cj0ZLxFal)*H z{*|K0KrUC^$}Qxco-6C6vRa5{)0(O!v=v&CZW=oYkM5vqWEJRB=Ru3(-c!&U zT#MO{PS{meq8>!bXeIP4{{uN!{-7Mu@_Ax?Q>A`d8RepsM|vo?)bFWJ^-;QCuShyl zJE%p}R{APCncWY5Q9`~>#7&_igAR_{^tY%T?^z6SJZsuQ%_&!mufQ@g94Qa`ENm3#huzBqZ2I##Qu zbd^fU<<)5Hm8&C3l^4BbtKg(7fUc-GdJvqP&A75$2$+ET*vtHNejB%!ZN>fouloS_ zpiZ~0D=sQpM1K_M~Y zwQ2fV&8N=Scc}G^#TupW*3atQ&|fnL?a!fT-b^I}%uCdC(%wo3m3KMVvAfug;7yn1 zS|BTXlJis(H)SF90WxG zJQL!>p6wcGN&=vOB1G_cH9E~(;3F|nz z0P9l*dfmdQ&gkknLd78Wv;dy=E$CoqYh_~>vl^u0hiLfxZ1#qh(wHQeX<&E{Fq?r` zoD7}iJu<~bp%^@Z>Ytxl1DEbC5Q=M~#qT7#L4V_WUI%T49l_Tvd^llA zc&1f_$E`oE)nLq~%OKAig6DKeDj)UR%CRUa9jb^891#sUrjy_<9K`AVB-r2=@QM?` z6kmb%lnr?8EATEZpl$Ibh>qWo)q?Zh`eDh?tE$5_&<&YVI497N)R?LW&O%vS4;Bu| z95}siV_HvxBiWG3z-O=g|F1fZ6wiK~KKELq@mgnqpD@B2Vy%R}j?Nrh$t0w%j^Q1R z#@H&=1p!{zMf=uTFjH0u+T=NG>))k}Vb_>an11Cs?!bXXc|1qXE1$G^}f2 zYoom#9Y~YVx_AkhzJ2%|H@^OOMq4J7+&joxX=E;<(7uKHqM%9bq80v6>|pP zpyp^HyhYwYi@AuDY&!PPE1{niB8$=8ITtj;d)!9m0u#xdq(9MarXV=g_n`O4pj}tA zdegh8ooqEWAKRE+OYg=iI2cM$al+vq+=@T(Ewq*SR(&LVy23eG896rr?GYj-T1D8m zbRe^wn!p%1t30+g&~>nO-Gkm)hZ5lxF9iO65%`hkpw+B5GEt?dFf^n#A?2m>xTC%y zkui^X55`p*7X#*486%l=U@n?f$zOUO^8r;4`M`MYoE1ioXI4X1JA`yWBV=I8Kq-p@ z*ZsLU8RYO7i&=0r9!XB5b1DSmylZAGg@q~|dW zsrF1soHCmueP4hKF`AQj?BEKKbiIyo1>KBMm~;owU8!p9N@R=dIN$CkUD5qmAD!bp zj9}v=)L{Vx{}NmW{vdw?B=V-X>$mEi$xgF~wGfV`LCDqaxBiAYoDRl9d6r^_@^0P) zmEsya%^=$|Gth#tf*r(-XDf1zx$RUt_6K+>->r$PfoIMz=76mr9MlTf{&PrPYA^ZM z>P=D@9iI|7VoGZQ=B0Yk?VHgpm{Q+tefd z&?PyDlq2KO3)fR0fw2T)qdSB+c}{ve?;Vk+Uz5eAHJvBR1N;Ioe|D5Eo_M7(EX2?a&Kzgj|l+%<|YLx%5GREK8Q(leLR>@nHUFIod3(ru)!B_ca zHUS0jn7rM$*Vk0);O*-rO@238E%4NB)9E|c2Bh6qE&JGODQzwE1cj!!3{=vZ1{ z?GQ4ZbGa7$XS$X$#-sSW8ROFa^{zA@YopEYu9o~g>KfBjY;6sara~|Ep@r*@mM*>U z1xj0d13ji%U;CmTquy~-nICu-{ZuQff1$zXk?)D`q=r<1dRhJ=<)}^c!SL?g0`ck$ z)!j*%_x7Gm6HY8uEv!oML&Th;)5R!eEJFd0{A_EvSoTmMbOt`MhGJ;1hS-%Q;Rti@9nBH4lsltnIEI*em#QuoWB>5*J_- z?c^vLWSV+ks_(R>pkW;$ZH#YP6>EgP#Z098vKd?{=7v}Vy0nRwvQ;wUMSG<@P}+(2 z7G=hw#qg}w)HrTE25lhLng=d^B>7;L)qjHEK2`lk{YSBsB3gTSs?x$=Ls9gT+DNn( zpC|22f@I7>{gt-dj3IT+mE<#$j#===yf70n{V&EuFnyU;>|N##<;Se@CweZ@F>UwK z!Dx@G$z8?XeFN8tpJ9Xa&VOJvoO0f>qL6AkZ(rc}pR1p1QsCZzy8&T=S6mYUs|SXL z6bi2w(ktMI(1I$awerpIH1({j;6`l05s6JA+nA>#T z9A&;ov*KVSNb4pOMZziYF(}e&jSt#y^{lbUWYE!SBH?+?gntU2m>$M@_3< z9%Ltpv@rRH^Pn|d7b|)zHW*a@+UybbHKG1r;;{zu&R$3IRaANPloS5{qdmq@!OIx9K8 zIaD&6&Cl%8FKMT8Pv@3NUzK$(cWL@DUzReL%*7MDw}2)vewkPq>Ekr&8XJinw@LbI zHNf!RtX5I3`bUG-S_-WRZS-uzRAX_@5w)Y*VC|*W54-*;@YW79Mw%7St(ssJgv;VD z_?bFT-N_j;h1@guk$=E6i>HP&L%Ei0CGH3_7v9?b>~>+V*uk;hNjq;k_S$=hUvL&3 zolH^uo|WLB&Lmp2ze0r}sz5hY?>dCSCp+las5~;yOC| zgmS^#0>*|hK}mcuwig*LbGbX@c+c%*DRpzktDGp$54pK{RGqWRx8=c(fF=D(rNGa4F(k*sY;^Tuv{j=9-TNHg$R`k9Tv zdbp~m8Ktn-yM=6d8JsW-swx~m^YCu(f(xGqPJ_;1<*D>Z_7=`(L%0I=?T*}l-jULXsZLC)am{gm>85?7d=#3<|m zVSrr-7#KhYB?Jb!tGjBttH4fX1y2e)8nGw(?*h|`JSy%ibfQrI@Qto%%HO}Mef;O; z`4+5>Hd(H48!8`NK5n2+QBVq&-e<`O(W znbdnao(w?a`f5znPBVXt2kiv|@&@M*E)=peq*&Ocu&~hUVPWBGBA!QHjlN#+U4hH_ z1M?pbogpmCTK%*CySmSNzRG>M?^}f*%RD`dc~&28HmeG=oXZ1B_gn+zz?70c+OwsYLMyr-%5m^ zc$Vh+a!Jmp+_kxlJx=ddpT}Qc*@1on*=$F*XREL|=)0&vpF=PC3SycAjmKmI&exID z75XvL13fhD*u2a!oPy)YFErhbw$@M^nX244v4b~47Fh;ex< zML)^Et3dZ6dkS7F&^r2Xu)?kIoc=9*Wj4G={-C)IZkqE*U1E?i2Y#j zfY8OkM?-oB9dQ*E=F_9~70Oh95C2eqkiV99g!h%toBKQWyYI1oid320q|31J++IGB zNo1A_61$&HvwE1@v`^+RGEC2>{E}Mxi~A4z^Z1r~bN%hqWy%^*I=11P$$&I?&ydVG zod3RHM)nnH@%Qv;JWHnFY4QQRJf|=(Oopp59gX1*<{SNl-ooxc)}y+xT>NZb>pU0m zC@3=ccu4KApa@UYyXd(Ed<91pYMZZ8#4yJ`YqaNX_O#TUDQt3&)Oo4x)4ODM$#P^4 zO^?eao;9*=Ey8{}!dAmR&gMm@bP<|nHqsB+SiU5;ncKv)!D);mS=v-ZiGw@A@0TwAHSZCK=MK%}Fy8ljQ0gBdk+L0Ka1+#2B|)pNdG+qd<+R6%5qhS;dPFvSGhGQD!qxOqv?%K@=&06ZYeKu}s`LAE`hP9~oY4-Whw=?wor%P$Z7nAKcDVJs;Qn69 z{$?h#{Xrnu&*npy(Qj@8KLxuc2ln@`Y>OPP9Ti>wI&M1R?5l;YEK_l99 zxTbDvidsN%NkMX1|1YVC)J^$UDzC&S6Xkx&ED+l%ZL;3Vn1X(>LST=2z(|`-CxTPc z2%VWz*~;wyaOP-@p5ZOl|41Lq1B=7)G~PI-@5BjaB<_=v+EaC~_D=I+K3EK_zCK7I zk0r%1~L^>B7j{;yN6$BAbpa)SRzN@8}Smru<_AcTaw+CGr zP5HrmZ@!yQPuM9O6~+nMg#O|o@qq1%{Vzu?r{wG#knNt19@3-8Y&Nld%&oqgz^{R!9rdR~wVb3Z@V4p}y$zi$daP zFPnkAawYZ_+nK8bzT`w|HyA*3;LT~Pme9+qhmp1khHf!NS))aRsx!=gPKj1x^&x0x z7tMGmX}y@C@Vix~CSndYgKEgO$E>70wU+Y2jiVt8P?xZ91^1v+;guDbKDfS3nJ5t1 zIiZkkl5L9^Duop^4L8G_s z^AGaX_J?_U`48rf&l!`|IwLqUJiDD|rnjJUM)7EU&GBeS8;{Rovh|+1CX}}gv}J?s ze+%E3uH0F+H?tSbZ1cceYh^%tRXc0zrKeI+S(o0cHT^A>>b?VLZq&hglhNnYTQ$r} zrk(l@*X$f1RD8iZPwcBRAoFFJ|7syM1W|QX$M?Op7$; zI@bMPJY{QT`)K=v9n1+wPv;?5vw#MHy#x9OHV?WGVuXANnGrEOPn)Qyu%xhJ_K$Et zgvoRL>vM(d-Z=-eCT2I!p)*INeg0h~qi}XLe^2#>zQ}k8>i%~$B7{IUEs1pcRI#8V z%@J+S5Vtbdsj^6MLmi?f(I68U>y%CEXSs^My;9CIOFH7`v^7$5Pz${NGg@bPx?V)- z39qVVQp_u}iFMD|U~UBmI3JY9hM0dGFjX|VpE38CkIb&-aH9+u7CWrx=5vyXr*B2- z9<>hl&rN;;pTfJvzihHS&VI)c(bYC{bH9SSq4 zb>MV(;PdJ$)^t9wWuuQ|6P=IovdwuHQF-VbI>MIjsA#P6DE&_2^ZZ4rUZz=@2u%qr}mI()=5yX))`*Y1wVAG zRg#p3F4GFVl5NoNGa0Yo1bu{!;aVU=+QA-TFO11Ue`j%|S*r!S4p3dHdmmb%4`H`R zhqub}JmOY(oruaI|8t&TTA5#zk(2DxZvg#o9RYAHxp11{}+a8J3?W))Fg<2ZT+wJNBvesg4xq{eX!9 z8{J0(#K2`i^@HXFKMOWOXNKm6vOz`M4M6XTK=$Xe)J5K{M9FUHl5a?M;hfVMH?p4O zR?4mIS?F7-yhZ=ZJL$Q<6W&K}^MkdG6WM4kgv-O8#M3QQ@N!%Enp|CB0NW4Ek~&T; zmBHRP4k}%QUQxT@%kd@1$1B9=-Vm_t+kwx7hN}H z#02q%xY_Z+#yDo##yhfY5zeW2zPel`pdeH3Yj(}X*~+j37=>kOT< zv{Y86q{aSaa)|U+?JeDxZc1yUwGuDSRI8|y^g_m0khPCP`HCZ{BnlJFS&SV`y_K1s zbY(W4o=v5JlAVH+W*6(0`NfO_gD=u(rB~4B8$(Uj*l2VGjSzDz;~6?Hmw+yr16TMY zXe41)BWo%&w~uhJ?0`y>OlCka^T7|C2W_4vHJ{#Vyj9?74)C3l!K}UC;AL@BSWL2Zcq`}1^)o~usC+C zVRI3L1Z-;>u8Bs}_U(N>&jK_|g`j0UOZ2ni*3$yfBZwj<-9hd#nj z@2xb_1G$jC*wHVA)-}ZPfbKqs9!}r0IzpE`i0}A&>Hxa;s#vS6$J7pM3Ca8a?&_E4~ zM~dSk*3V;RZKUrN^e=b84(|}u%v2=DqLB4j@}DkQxm%z}wxs8o8RV&z1V+(EtOdDb zt1%OrW(twXCbGinMV?SSaj!hV^?pWK=HHf|*y$P6YpXwYvWi8J+FJ^8X{hxHu7uB~ z3sjLzqZD%yr@i%Le7nQdognS!P#;U1ZaeB?`p!l$y73L#sV{`5gT(fYuiWB;d)(L10WbS?au75PB&9q#oc z;a|BsIv8e<0a_tsw<-~wI>}461NllVF-lX}%n0TaX7v4#{fp+OFz2nsMg@Kx5y%ka zHv3p7nCtpIp)=i8TStdki|OjF=pe_f?Lqqp>AQ{^Vf_>a}?hjueq8YBSf+l$TaN+`w!?Ow~=&t#CAsO(ml2p z_=fq!hfEv@TYcI7Tt0n}*;c&IK9=I>ty~GNmmJC#cjg!w$`;QZC`nZZaVv$zT1Wi@KR~dn%Zx?nDauQQ zN-c>6zw$KtNT-*vIQc4_KwEfdJW;Ch-{}_S zZEn0Hnz{;?_f~cV)`@xSC8{DIPGdY@z3{O_Tm(f62DdpTp2aiixHSsd(aBZ^!kVGFffNCFSBeC8GNnmGZl`7f#|w44;UMBacNd5*Lq z^UQphWAwJFBZV7peK&{U{MnJc$aH4caewd(?<|zGHMZ;a>dy7fKvz3wQD>mLK~Rz4 zpMk@mq$LCw3GEhIJ8qr- zSF34$|3T>`m~hoVA$iNxu+_A+6#mPMrWobeD6Ti2iSOz9Y#!0r_6hvHCY&~sRTxa;PvasTw1a;)TdQ_?>-SRr&k@U+2q!zy0o~pi)(p=wt&oTca zHBMWrXv#}vDDvdbObXQ0rdCrUFXmu(%$KHQeg-=((9lpTe-cxbp2#JqgC~8S>Bc^0 zZ}1LVsC|g@YQVh!S733lVLH1P1|@_P4qh32Ep!nW9Bm`^L2%>F2dvb+O*Z7(|vE*Nsg1 zio{CeqxH&dQ4$Uc zq4xI9ILB0H1CTOm2aFELaoSy`yI_cxXGq@91$q|gU2;(=vBb=LQ9+NWQobD-iRtq* zKW22ye4Sc1`R=zx-*S?!ru6eHF#7VNI04M)Q;q}ngW^^$!dA%<;5gz0l7j86UjQL( zfp?^LlfRpsSLr7o_18k%=1!%WRs}6deYhdkE$uthvxQn~t+=)ud!P^U+uZ!QBRq}d zuYM6zvhivkeWm(Mm2uYS1;+kz%E2UKm8u1|^;)#uS3q-1B6E%p5Jw7q1($7(?H_x4 zd$N7EgK~~{m2`CrsO$a|sDvXXrC2vWX$Rd9mY=KM{@_b!#Ftw;v@3v?D*B{+Ru}`L9*UO7e)L z6`${XvA&*7KV^*Ne>+B?D_agK9vl?3-u{Ij%NG=5MT2i|71ek87i4`+8<|l#r;JyY z?@D##{pvCJCGMMXY(dayM%exV1L6$*1HRS4>Kkc~w}w6Upi`&n+PuYtFv zCTidGOCT6t1efy!^NHe_F?@0H0MbWG*aWseJdy$MFcqeg=wPZiDMG4JZmK@x;dK6% z?WO%hz;Aa}(AeN*VJD+r=~`?A-Is2_f8!HamU~NG!u+@*w*-4(4Lg`P%7zomG<4&w zV;Axv;z)b6y?}#tba3rLjhVR!DiK*3j=8m5}srfJV_=W4m5kTLNcQaq__mL<;#W_TKmC19UAq z22HOa`~vP1yN4l6F>WnCj~^vu+wR&2IsbMo1bOIl(D2X!;fEtO=4qK{TlAv*bqkiw zw;-Ah{mY?|mbsZ3+tO}kb^hZ^8=HRV5B>Y*uTQBHayqDY$X33d{hqU@Eu3>;{&7+0 z1Sj_$@sYzOC@{~Tpf8tuegK(eC%Ra(nWgxO4 z)2MA`Ni@w2lneR}_{REsYyTJt#!Yphnur!Cr`ZI%tcT#drp^7O+UobA`bJqg49&Z5k6q2J08=<5yC_<+&RNlFmQHI^B`9!7rHyF zY@TLOHS^YwK9_%D^uqA!LCg4G>N(FKUzRt(H_3a~6XV&FJ3ISV)>5xaugPQxO~kQ+ z%npZ^HJgD_NOGYvy)=%=Go+`UULMJ}+<(Vg+xOI8#J>kqwKd8Rl{NcY8(4)s!R*I; zE{h(_gtKqqQ{7Mfk477+l22U;HE$0(Z70A>G{T&#|4|F7UVSD!yp&bSyiFRBnPdai z2~J}dvlDaw;npQ9pE(^(NcG4_u%NDwHtc88D-k^Tl!5cMsui9`f06;dJoTn_C{CjZ|%Dl zqfay5=<#|jZI|8)ZlCSiGW`cwKg{k>mp2!i?TvnBE*gz_^EBKs#W4d(ux7!RJs90?{oxahp~s^m zekgr|X~(o?zu^^>kk zrgRILj7Rb}`H=ETWwi(`9uED^#$@<1YZ!}+Og&TYquVfX>tc$SV!bnVV`{nyUXTJL z6z{YICNs6+g86NhCB;ZA{;R#_Dzh!QY<@DE5e+_|dE_zK0>AcWoVp{Cq<#e_bV+Li zIZZ^%haB}55`tgpBopBE9SpA28BEwU_|#9q8Fv(P6$+`OptP+^zL?MwJ54Eugl?+mT9H=!Kru@)G8C z4dB}TAErAgBmudQ0q8|oZ7wtynJdlW;DyJc2Rg-|%oHOU+RAIJ{Qsh5M6~8wNme#h z7Sy7JOiA?1A4StvOQ8hT_)oTR_I!>2XPWb(YevA~fOhUb?xet?L26)h;O&52&UTKk zwtn^jHbD&HE3ot7zb%3pUpaEr+-V%ue`$H(d2-6Hk)y2bZ|Se@U+*93f8yWkSN!!P zN$M%Lkyql**n*w=Hg&LSs_T$^NYE8zYNO3l<_+_psTlRlBRFq&!R}!uGX6s`C+tfV zhKsZ>wUs(et%Qe$h3B^y^$1GFSkRQ3API%uQTX^}eDaf_&HfL$sQ>!f6O9eVGUEX5 z(^*(cX5;5FNJJz;kB9^i8~r=H87_09 zSqQBuBQS@3N*0hMqzqo~I{4rFlPX|8t%nL&3%>U^gv0!E7-m2wRKXPJ>S0u6I0w#9 z8!`1OM@8e>OrrjUyS=$p$tq@rTQSy7vL6a&12lOM^RXFgjx-yZy>Yf5ZEiuE^jk9+ zj{L`@3GThB@J2h3fLY0uVV6KLY0SOn`tuLbZZ}&96%)k>TW?!m+Z9`g{gD0Xf2UnX zPsd5eM@KMP1DoORhK|~HNgOCX5T**x`F>m^r+@?f6WypMFu$hAWZa*Z^!l2mPF0#I z_2ruKJvappO5der(mm;bv`*eGr^>;~40Ku@Q+0Km7N_ajdEG*4_yp!YUClG*KS;Aq z#p{1&)+8nI*%!sN`buhH%3KonIgR;N1>70&IJ>VxK4dYR^0(m@xP**`VMXIU3IM@> z7BU*6F&71!h&)FR&v4L(CgSg2_&!GCcWsE7av^ktj)AI^1$S*Lc%CA0A6~~K>J1&s zWHEKwD{MF%rXk4XKIbEaM#42ATx=_j6ywEd;sx=dctZRvnqpmBC+xE)*oNBH*`C;L zfIxQ*`}0-c%QHbLrn1(=}w#|I;-hQLHKFs$QE|y zqcF+nA?26z$;0Ji@=ZAwu9BK+lFHy|6s@1sqm5QZs?pTEgWYUX+%cy?TdIP5%X?%? z>R@*M6)vqvJYigPL1cXL(iS*n!!TQK4`)Vaw2hX){;2@k3wB`wIUBRg0aPst9~E*j zx2;21KgL@Vz^5vM9*-dCBjqqhFMz9E6CI^9ks-N`w37>Sli5@P`bN``ysv^e@(v^> zN;37BvCu?LF$H@E+4Lu91kA(i$`PjKApo|Zo>Vo-pg+4+RfxhEr&QW>IeP#kUYzN68WIh=sg3k9Ow&`4d&ovlMN@h@H5v$kHBivr!m<1vv$n-dK0QA&zN@TvMl=Xs z+l)m>sdU4&?qzKPLGux2Zq+f3K8yYNBq%5W*vSRZukrI9st?nZj;2}m1$s)4qs`Nz zJ~BtxR`eC%gqG+GdzqK(Ur=Tsj{ zpidb)n0aVxsf!+*6cT~Pp@*2IA4VSO0uzYTaR5CBjSvsXYN#kp;dU4T#&jLD_dZ5m zHrhIDRb_waP3hi9aim#!|DTZ&Yz3Mf*>^@`;v(yqf1%P5km#0^dDKi&kcqWEY8|** z;HEVPEk%b4P!@Civ6f1&GS-1_wHpC}y zyra7_wb+W*Ra$4R($`3N+hq`ut8!x)7kz-;%`OAsri$>L{Q%XgHQf!=v#wkk?(I=l zJi}7|Lu%(X)|OD>qdMd6J7^*sf$L{OVl&A6f<0S;IZywFrZd^7WKJ`Vm}^L+=>m_V zfVm&{>S1d;^tZ!sA|D|UEHd~=nhm2>XcSSnx29t4*$y2x0V%T_@b>mo>)?XVfcjMe zYgab%{8!B|YY9~kclSkd#vB29TPzgl3&swrsde7G4>nUlB-K{alhDl6lsbY=t@)%J zQbPYT+B03*nc8Zs&I7RyWMKzAo*iQ~M;pliCNIOAhnSpWf}w;h z7Fn?RPP0+$dNv;@EHv)1He4F3v0GR_YjPP}Rjx4mjK9d%6jC^im)SpTZA{VsFwL0^ z>Mm0hCzMS%pZJWn=Qh*gHXXBRSspxymIF6g07rjh5aHKa1%fy-DT>DPG7ucMS@y#~KPgY>+ znA`Qu#!j=5z6K2QROOD=Ms2T#C@bWK@_2cY7NpKZr|wtf6I^4x^uE|#2Vx3Z6Z_(8 zaER?;PTq(b-m2u*G>d~4^R2F^u$-6qUqdqbnlp!YB(&|_?8tFwvd%AFx>wf7b| zutpo&TRSCtM|*R76qxV4>x**-{1T^JbmOjW1XHwywo=#OkvMMjSnkrDm$OUr) z2)A>H0e=^mxTLeS3U1XO`2T-UsqlB5qW$!I@I(q>{&`Q>FYdAB+D6(-Ij%S>x=IIR zIIp-i1T1mSL1SafpzVR2Xl zRXK54W3pCe1!k_v2*^I?i)ww!gtm~}f zge^t%z=JiE-+<1f&18gmTYsa(N+bO>M9 z=CZ6cS+CO4(l2_OX)W|ta-uv}izO8qH$TNT+qP76I6Jv)&UN+-M_$)Du$JrE_90b2 zmpMmIr{_a~v`ICj?tb0d%ri5`=}q+-JRkt#T|-kao19u z0>!mxf#B{^G`L$xLflraegDnZ{6aUod+$9mXU?3NXP)6aE7+UcI#_#KD?6S#ZaD0I z{rx-v$$_PUw*PXB#T7_6{quS9!sPjhjebTZ{^kxOa=wh*tE)mIS+qa%U+3KJ*V%tX!0^Dl{tERa z`}lA18*OW(eCFJp>7A+O_MUZB%lMEwJAGqnXzIPBUMXeLn`P$A?CH8s^@Z-*FGdU* z)HB4tsAm0Mc?A}0EYkN$1Z!hd0G$!Sspp2a7T&A+K4~=d2eOvVG@7^EN ze%?z6NO_(7F5&9W=1F~AzWi{1SVI0(p2#CD|M=(h-{veIa3Y|rGs*sswWMv9b(5{K z#VS)!tSZGSBbcFVyGgkLI{bcKaA~;RdO> zyk4Ft=Ts_MrdzIAR$J;)pQ$=oZ`+j|*4eN=IBd=AtL+^eAN(gc-#fPk-U!MYIwjla zY?E`0%~dh?>g?IV8U`=5yc2SJhJj!3_lBo$PnwljH@0Q$jJW$hdnI*A_&xb;YO~ZQ zNu5&1W{%N5fFigyPx9x?&xDXY_Zv9TPIo`QCs;1RR?BzF1sGP zcVz5HdzW51<3h&r)M;t0Gpo63y4Skj!GKX$4Fpk=3w*?EkVuK3V-Ax^++W;Ho^m;% zy7?G%bO0IV&GdolM{hN?9l1suwBuMaz47quhx@l5*pj_)g9R#oDD5o&!Ng#PC4Z(P z*?!!S)6viI*k0Ve&2hqSxc@2V#(?aB+k!R(?+9Ivk2fMbDEq%57XoK1xup3eN1X{S@}CGVBDLF31p8hkXWXi21IrZPnyPlcsli(YT#YPa7=NejMJoc09 zrR@7H*{%C62g$dqZ9S;uqGEP8qmBNncc%9+RiSj3f9Cm&+Ueudrh&ZMobg9SQsyvf zK!0)X^A=G<_2%S54HZUGd$T)yv2HTNFPIgDFJ`hJVOu7df9VYk5gb<(CxT?oyBm$= za187ua-UoJC_hvdC^IeBZ5DfsLwA&Mxa{Q}ul-W|zd5%AJaXO*xJXWCM97ffPGKvu z$AvBo&ExFq@N#u2&VDD1#qM-ZmCPM!YSO~wX{jU9nx$7v?}sO=cJkV^;u#IRz1<7E zZ`BB`knun5Z*ji(S_%Q(x6hhQ=}89kJ4Lp>SB5FgEnlRi=1Dx3EA&0uWBRRle08A1 z)1B!*(#NGQ#tUM1Rm*&n(bB!i+gJUm?SmmF*r*GuK%!BDGh#Dw6{pc5oM`^wj5QuE z#R=3JFCnIaeEy4^bt^vKLDE>Vae}D`_QCSNvfWz45`hMMZo5LY+n0U|{U`Z%bZ&P} zcRmQ%5|}OcNpP!>KEYj_TO4;Sr7ccsQ^kqzz>pvJ#<^=?AN-XuATvHAD)VZ_5Io%D z(;lbg;#@u7HOMu|RRVvmT`i;4F~*UxVFTa2+f3jTwOt%1#VIY6V0oq3l6d8>#uXUF zH9YpesI$Etypz3wp6%{>?h~#=_ipzacXqNpd*cC~&q=g`dB$iW90L_c)LHSU1 z;|A6B!r2uba#p?!cCd+bgRHnC;D^_VEyzz%EWM=b;sH2;dXc?d<1{DKsB)!LTh+#`pCb6G9DuD5F)WailF6M8&l1B#u9xK zHPtnp3>vE3=2M4yA9`1+t+d?i$b$9{wOx0sYlxwr(=$0^)HBX>|t|+ddB8L3ywsGvPgGX*O3234P!j znd{vx-Ovo_I($cM;3;n+ACkJkBK`m-g)iXa{5ki}g~`_`eG`T&V}y13YpI^)q;Ns& zZHdNlMrQQ0_qe-Z}^Eje$kr!GNm8CEE5nHWKSio-h(>$REios8Xd&ik%TtevW6bqTCf;f#EzXRn`9$HXCQf?PQqT7uHbU zNT8l_335^P8{dTIdL!`?+!y7{k5my46#m7l(^e$AL8xiG6~E%)8H<;3iCEbD9gOR4 z@)<{)^+6Y&2IacWtRhVW2U!Sok47G*}#RS00@nX>>AI3K5)h7vMcOWopJ-JU1iYK%ZhvGgYIb@B~KsL&j;W zfk1M)is7ZR!o(R0&qEO2+g`*eADi33dLOtu9|pK^Fg5SYO#LVK^b<3=p5U7# zo2|*CQQ-L~i8cGoY)h5QaJ36fvi`U<1< ztHwS37&R@wAU_2_Ozi**d0G!f229522sP4fGTJ)a5Lblu@>IMj1B~kul?SAk-rK?| zaKr7*OmAiREKKz0wCd&?v6;L>-%12-oMjGao0;oZ??bH?aIe zk;c<_ORI{pjQKlZBOKJbjh5mbqoPz$$3I~HBDjRrAfYZ1Qywnf6~?MN;q&-~6T)S6 zxnvV>3MpEBeK-}zewuSUQ$gI<7hk!<&9^X@Q#DYm4r4(iKDL(Ru_sXNq62lJC1I?V zL#nUm!`n8%aB2C3l2RAELldNRdJ(wlb1FT__j@7uTU!aA^e38&7{oxv+cX_uET>t% zb3%O}G*X6`m5hlXgL=s&I351f=GfX%!7w-Xswqn9OfZyVr25)8;V#x8IrW}L{QqB3 zanG=nl2e6-FkK(EyucJM?Z{7hJmHSNExiz-G3^J<^peD((+E~Xu?**89^3n z=y76GWr99R{^%)Fi@X#8!SGKNzwW4x4S3Omg`o>kso*ODWIJYf^7L@ zD_qN7^70eCf@O{JpLmo?!L^K8+G66nFQmJ~24@Omyc>v3w*;M?+eo*aa2GemN;BZc zPEe=mHN?aAKw+d-%~RC7ohs5D?X$(tnxA;aIzt=ei6rCUf|%dZ(vrZb{w7QzKUB9c zTFOm@xSskO>4#ojZE6h1e{@HBqYTqeX~T%sy0ju@1IrhoDNO2r>3!6AZ+UUPQp>ob zloSo+2l&&o<~gmj_NzR~x>8PJzGsQk)v;nZ?nJ#uz((%|8qYlJA z^@b;#H?MfX`rWF_)y-pMul}W_!mw0GZmHBGhp3lvP3`JgYMc^BSQKYw9%k#eGo>~0Tx z+{=EP5-8BvD)?0}W2E6{?ftTJ|&i!6|qa4vizL+ujzDIr3j`75Y_K;lBblkrfF zi2EF}B9_Cr)?Iv$SND@J53|&evGpr@f@Cd1@6`gMv9(H4EM4QJB_|bU#Y#+S{hA= zV9LTmP^pGgkqWs7jY_(oF<&nWYiUO^Lgq;Q#G_<`?!+TlTAYQC`!&(bCdPl%**c&K zdO<#^FV&XbW1no5u98h$9Zz+V*+%LFvM`VI1@DlD7-s|g;+4&|MCCS+%`m_S&|f|xy=xK-zSYo&H?lFjo2fH z&AP^9DhHK=nQ<0*BmH16p~4S5ba}9-&k|idBb*?jGgcT&Kd%^*i7^!>`V?*MHZF4F z+)I3OK4W>?Tq0QEYCWr4kk^3*Rd0;kth|QgqiFgLZ6C9@LcA`{g7>p9IaU)G@9Sn^ z(T=?uf>*g5QdtFk)yurcoNeK|4`^uzxYJs(|3@;D>&zH!4VLCkV~r6YlIbft8NpBx z1CRA?>T#{UaY5H%(d{oS;OyBG9?&jCS(fVy`9>>a9?1BWL=z*-Jfc^aM1`FN(jB<$ z4r1%}HpgR+7AL=>AhzU5@f-f_WU_Utn@6aEH_;e^EUx8>vYa^HP$xEz6fLwS>wbx(B^9k^i`&ZIO|vAC>36xfV4Nw!Ez0v6eE!;K{|!)h?m|}IOm+e zHQ z?t4*$N&b}iM6VC4|3h&e8B@98SgQz9qARS+U&%}PApIq-5w@$jJ(axmykPf0dVH1c zAm7QBJw&9*_og!GOZ5n+v<~KVDn%V87wZJEu{+4mYV^QsW+BWdiskAjmL=ON5D#l( za<7gMdx#R3ito)K`dK2A5A}34KiSNS!9?^DuS+q4kGv*c2Lo3r!?L2 zk5WiZmXDB6af;aULUiFr@*AqNa*pCxJ!rHd&+vtOT<&XqFHMDa$W-U4UDaUkd+oV! zK+3CZw|uc3wf>OSbK+Pgbk|p@L(~kf?yawh;#aAXQU}C>EO%9+#jTvHU+Wfii}uyr zEjFa4kOg)hIOC-rFces&hOD6oI9Vs_i->VtH@_1Tm=7b)B=QD}idQ&mt3-(h>IcBS zoP|fetsxOLE1|rT1C-TLHtL-%Vl9^j*ME{+z6_zGQcvDbhTJl6#`j>>Ofp7+aj=-% zvHq%}gU4_ZU2i^8`;ZNqTbrZp(k@eP?hjUT4S_6lsj4sx7Kl{QAywg|vJtjW%_yTC zHa?TZwMPFJR?&ju6LAnIp0Dyinr08dBlmxSqW{U-j7|nFK6l{>?Pa9eB|^V#v5@GEuChx7AES`^|^X^ zb0#tWOGZ7h9{#xr(nh6*JYDQghT~*&pPq|Y&IN6=Q4CJ2F9!8c#7R;GP(=zpz7FCu zG12_PF!d^0tU3{-Rv2smD~woLcb~eFx#hZ2H*gYZ!dqf+PNI#GU=Mr|rRvgFF%h(u z9oCrja6+yIr@Q$|-hYIhBlPBr3^;`Ni-PEIDPdF#+!2c5i z-*F|`5c|My<0PZ3rkR6!Z7so%Y~h5olsWk%HU)=r9-P84kW>+(ue#nD5NSam6IU{> zzk{+&#Z&c&bKntpL=&k%7HR%1OolZz#Hh?JzYKhC6zpbBV#E*e$&XR?%9qJ6ylv#y z^Jp(Y1D;ZUXYYwb7p6(7oDA!GJ1Pzgls4n>X~_vT9aP*FM!7NQ&Kg)+P00zZ4lbr7 zUPA*^fCUWqP*XP6X+cK3-rej3KX3tII6ADnBEZ^~8y)$ldCJJp{q$D)MXi)^(hL!^ zQ8V%;Sjha+Sv*LE#pPH-=RnB30WEl2o2CaKVfU#NPF5MdwK|*{S4suMI_AG*K*eZh zwPpH81IMD+6^@S~IV z)TZcXu=lNCI9f|v#75+59yRu9lhhEh#&7YdLZ)0zY{ZvvdtD-9y9C+FF~)6Hgnwb4Ry2SkP zb3;lS&qia8($(*M;Qe4*&?9F9$6F;{XrlM?esFbh4RE)EIpYP$t!1o-R`NCJI9BjB zy`t6|CZfvJTdiVD6}Cwk@*~C1QcYPP2Qddn^+a&qS3RFW>6_{%eKKCk1i7B&gyjhs ze?zV%%@U4N`?L~h?f@)@KS5EwCKm8k{6pFcJ4RjfTV-(}wcSR5+mua%eEd`7r0)eE zTnsI+4U2pZcFk=xc5V2bI>3kc0xqQ$pvWeepJA>YWmLzi?TKCdCnu|(a63EkT>1&u z^?Z1qo$O{&MAEjfF2;~CU)a2?XM^kQfc{7e*2bup)gszF?WQ(}4DnCY|D3@3>n!HM zOFiFwU@X?B=$rIj`Xx?JbwTXUlSazAJdnA2C(kBRe}$AHJ_b)4DNU81!0;j1=Q@V^ zA9S{(O2AD2dVZsbtRc4dClt}nCocmwV&D~ zeYSC)eDqK%56*^-_^Z?cORlR>(+oE{lKDAAf2eQ6hd+zGtgMuPJuw5;_=|Y_y|7IdHQ%(Gu8CTs%#_X3M1+z2u3VDA252f9T5{0>fqeL4Sk7O5Re z4eADXmi)~Rpr^+ela2p))&Neno$%UwXiXU2-EC+}H$LtiV2E3Sko^rC>Ju@9HDm*Y ziiO}I(9m)ph^$WqP1O%ic@KWdu?JS-q+^;?WD~9vkLbmxYv{#NK@fD<+$(9#)FSE# zDlq&-Jaek2xcXLWWpoysQafUXb(8fWTr;wi4;*ByR^7A4{hfTl#@-y7OLyS6S*&cd zcDJpvZL)5#j8`_xDb!KzL-gdP@!r^NmKGZ0<=i5_Qo^i(e5<{^kR#5%(4pEZ*`q8M zWvk$?xAor1oS*g~d0_In1ROHqsfi^j^*bi-k^NLv;TdcFUP?q(tMRTgO;-!9gSlWI-ON=E&f#ytJzr&xBVkOq z$w{>w*zh(W#KVxf$5`kM;i=HUsD>j8OOOj2oU3y{IL{{+Yz~paFlK!d-|LBoW-c-p zLjv{M&mDg?hhtrDBZ~MA z>_ss+FZy%!LSn?eNO>Nxzz5BdWJ(k3Pj<$&B6SM7FchsWe;zo#5DC#7y_)Ra?&RHMwsv;|-#jwiV%u zVzi?wcdsH8;;E^4qGCbr`^U`+hMLjIwN`iAA0-QoWY>Tu$44dgwmyKjGhF^kXM&|CehI(YFiK;P^!EKd_#@G7|Tw zxABBE_6*th$~xTx)q)q9x`7y0} zOZzouQ$T)RV#x`#<`Zl219F_eZ2Efff%f`VsEaij&&>X$4>!3s!Tij<=9Gb#wJ zpTRTYX@P}(IzQUNKz~!|isw0Lw~bstA*<~oPYYrOsxUk4(X#cBv>vR{i9+kF8SKiu z4PqtsVs4u=n+1^B;;j6dto4@2WL@6P#e9Y%GpaWE& zh|K0>Ef=Tf<#<|kuCB!Mv$HCGXErOcekL<(PBGO874nO)0^w=S&)F`~fb9k5{^I5d zGLWy+nkAeSd*I0(iEp62K22|}H!|J$NAttU<1$y`zs^sTu^1Y-p!gp<(tff-PN50> zIERcx!VG3fW!{s>j4FjLKa8Hvhh|7)KJuc+Z{d+G0)p@<`%n_9*b#WMf_Jaq#*JG z;_(EHBaf@Td=i|cL-|WNsa#eT5|hm(wlTk98M(B>oE`dVGqp3|;~e^WDuCs{OI=8> zU=(B>Kf^QJ8!g}u|JQC-$$Kh_loUsZCGp!fBce2rad=FIsmT7lg0 zU?kgad;uqRkQI`|I9)+M+(nv((~lJV#Cw>hP$D#Y`I*VHd>Y^i8e~0LNp%^C&gl1D ztcD*jBi7Tsel4wh=Wjl1v1$yxi zi)s(QF|0`kIEz`+zpO}YhwI*>2D@D>fgE~|=%rU`B7#b4w2WD=|4 zA2iQn+7OGp)WV`^$gcP&v6Nb@of)i@My$-ySYpGOsY|TZ0?6Vh-X-rv*vj8M@U@je zQ`wRFC$yz{)-F?r{&~=44&*csb3KXOwi8llWBul0mHM%EUo#Hnv-VsLX~=0zmM!uF z`G{h7xyr8bkltQl7WT74h>Yw%tZ*-^vxXUgJmch<>@x{XvkLc?{XjW1ZO zA9?0Zer~g3cJbLU^xy|(CJqhxp1niCXYn(upHWDKh8}&!cfRr2YkF{$ePc1Rv<=<9 zmDt(e{C^vt-=Gzde8abAzMw5SS~CyruYw+HiQekNDAhtD{x(Mm*YNM*UDBRW zvoUtxxFQ9ulaRIZ`_@x36`JnQ8ZT|DfL-7~8)l*%ecD}SY;U2>pJQ2kW`s`DvN!bT zA!BnM>+38%APXaFXM2my`i>pqEp2;)=6X#FJzSZ@>_}*kpS<@8%l;K}m&9L*yiVqw zOkO9mwj5~Lc%CZpDx5y~UghJb77`zXOax&K)($=rIl zPj>XVgC_*C52nxqgSnG=H<9NjBdJ#Q%`f!lC9VF%zh97dpS}5tv3*Rx?ju3>n1?gW z*Ev>HB!AuFwa*6e=|%<3`4xHdRgj9|ia7pnW&fhVjP4KmCGyHQa#olF%OF1<##^Dy z1+gj0u`+A3n-oK;ilLz!F=qv^GfYN5ggLa*YK{AUVI=SKw8#AaJhJpX%hvU9et1nw zPczby$X8@mt8TMy&tWk<=aV#Q&n*|#&kcH|(8F$-cnWk-&y zGkPT%;f}1w{~`On|4(A~sz}Smv!;EPkD_sDke69^ z(CFhE#xk1KXlGeYg~~o>#5Qn<0+BX8Nw8icjOnkQ?97@ux$38 zvlQ@#+f;Xqt)6?dZ2+jFGoBi@H?ClN7jGro0{2ndQg2DC?CENmp*@!VryUR{m?83K zb+DwvKKNRnEB$Kx5Lc)Rl*Z}@WuLc$IG#RS7WNXqJOGDYB_UnSE|xb3Dpl0yQj}g- zd?EhuuCk5LI>PYj5~H=-@D`>JUpl7k5qAh)BSL(oS5oq;Kcr21bLlv(PznjtBR$k{#o8-R1-&=i-bOM75%E&T#i#`OQZB{_!e7B{>Bmckp3su z(lvd6)LbYaPlef!d{q#E=ZrO?4ogOUdAycj){;W07xj<4Tnmz7^la6hZ0S;$>sasV5PdCkwXkX#6(yfv30841Ec>4?@B%;tH)lyx#2 z-$-@MY2rPdj6Tl4JI%Z5LwUOStM-RdP45HFAx&K-J;2JUAdZKrH4^Qfpznt{Zw219 zn!;hJwEjjesBJW;rb_f?vp7I&Y3dfQccQ6S=X=gu3hO<@!p45-BsNtIrHxQe-7Xae zJEaWV)==(f`iN#l+FAaJ)@&b#}_Ev3!oLLzVt>{Hy`s82jot0~?S=VHe$ zSO2HDgjoHdu+_L}-UP{Wz}%y+mx8c;XMsGOYqSv;Q|ktV3^q?ce7FOQO4QlSMP$2` zT$6YW$VPLE9APA@`Q%*^6^}Wo#SDuakn zEj6pjZ$K=Di1FSuWg|I)V=eF1$KqJ6xNWUxj-1Ebs`Zt-3kS6P(lvC3Q*waw^(db_ zkHsjnoIK2!poLlfR6oivzlgK+czKT=BF-1my|aW?icMOs-x0cLhs9vQF0}%w(n4~BR(Cwf+dcIQySjc@V=_b>RI{e+*gU7aKQ!Nwm9K5#0?eJ8ff zPVt%0M03jbwGChrj!ISZMc~K7v~E&)VFop;qltN6z+>^iIA!jV`U-PY(b|k?_bXa{ zOdk%5qau{o_sH|Tg~h!J_$X3Rn)2lz{^%lbN{w@~b24?KNjV~hTv z`5sxifw&o@pRn%H53|SIkgzWFWSGjk87V?eY9!w>YMZ7JE3QBzS@lTat&}M`y*tFM z@>MZPKO`3y59=S)4pgw%OD$qp+s$F(5^r<)rO;2Ghz?i@j%BNU-TYIiB~>^4q(-dA zGTLN(?yo?WztjuMi$ps)>|tUd{D_r>9#UDey4F~{D(@#Iz1RF9-Ua8i(%dCR;b+|^ zEfTL2q5EnCfGQkqzBJ0hnmYu`#-H_Y0!`6F97LvbqER2lfn=~-^I&>Xm5NsY%o4A25S)n5_tzZb}_LOXr4CMS9P#$dI;$t`4)(y#EaMzWjSx`=3HEl zsvJYff{p}<_CKKw{Zy!BxXVbQYT+OvqgEJ!?M547=l4MYQJp}jhF4>W5eMop43FGk z<0_c0eR?$_QU8)peH9e{A-vfyiLZu>Ir#fBy)MF8AuqGBUbq4qYKpOe*wX~GcVD53 zdDd8{57PG=SRpW1-vLvVgOgZca^4<*U7Rdkp+;{*@elU1xl#|Jv4hA0bJMfp(m8OG zd&B~u4n4vgbVwtysCY=+g6$ec93-2#o<4qt7j7FIsof=64wCaJ>0pgkqr-<~4R?7b*&5$|BnNY@*0hWmlX<{9VdO&*t1>jd*t z1O28t5RR3$`UuXz6{yKRNlPY+y}VXc>j5Uns@{f0C>A{3b@L_>wH3rs>zd6t|Algb zD#1=(nls*E>gS$P?otigC7-tRw|24Dv){6PcZB#I@hj!`$nnbY#BXRovU9Zm??J-@ zp9GBvUQ0!UmZ4!ehG(A{n4y$bn|VfMI@9N5oJpyhQY}6yuI0~rF+bxEC2Y(b;~A-c zS0{Q$z*f+mdeE(GPpyp=m-PbZfezs5M&kAPO}GR~JJNlEs)Sp-+0-S}s~)O80D*YN zJ8>tErU*2=rdo#BlpFK;Okw}JW+-Kq)d7_SbZIf@;7qLEXL@Fcn1o;ThI zn1ma`@-zUv>n`N79&z+^`3y`mKgCDlVepJ^!YBHFMSqsckjgvGGG|EtH%pPwa>(|?_3b~7I?Q1Qk%yGsv?_l?h3`eFt zZE)(Tq<1MvX^k>kdCqv7k}Gl>DQhZKm48}lTW8q@Sj$*+d8b&={G!(I%yiAnSPDP= zh>Xy*4jJ`bqcfYi4ysSdPmF?dXp`C07z8G^g#1M;!x?j@ncpk|&rx|WdrR?fIgN?h zTlI}vQ7x!X0u@x-SOsQ0f^43zWXN{6e6lTclyx-nE9u{aEconx{?4v`&HWYUoPh6v zg@eWfMh9;U%bO=X_mGIcatG#32tOC38i#&{ehB>B_WR1{T|a8XE{JRNBRQc*a=r93 z?xphd;Gx;m!}Q|@zivGPi>PrHoZaW@H8Ric3S<^0M`v~sJhvlCX6=v z$UiJ!Ew9Otydl+u@oJ>LU;VBx24(Y<3Xdbb&()4vBRw2@_$hYIcdC|W*Z(jo!KzY> z$mtN+7DkGTC987R8epsH_~TC%6z-^Y{BEZZ`!}F_W5|+FR@Ev>ZN4M80USaw{Skpkw2fO;Pi-A zAyG;&_GFB@IAe9v^Q8Il%@Y2NuN>duXT_i4sg=CNOv9S!JP;a|Jvr<^XtZ;Mt+~=y z-e6qUO1i41I#cE*y-(_tyg7AE#)(WfmgW(>U`2>UJIqQ{&#FjtYCLFAE9^^yJ=-$A zxn5_+d6(e%vD=UD&ofvMT<<*c25Z{dO^#+0mI`d!J? zLgzwm>yuxNzE$bUfcvQr7rYq#<4Z#C^j$_h>vH>F_8Q@f^PMg_y~vvaYr={uUA@00 zZ;y%naP-~2kCVPd#UD%En=vJ$a>hT}5M`FLWXONv8*@2wozK2KWUE7zo_Iqt|4Lht z(l*7K>`mO0s3$#38Js%Q<@QE_{~s3kDImAs2+L@}syV$bwW{Y`=1HQ$|8s@8*15jB z+IW|cS$|QvY@K9tTVklFw$n&Zhg0EdvuA?0yEm6targ7)*D|!#uvwiY!~R$0w6!gH zo(C+A!5#L5^RFa6L1MAyaq&O7q@}Ah#;=DnE}&TOiQw9yud=tw)u4b`a9FVhrJk0G zD;AaaY+#a^@TT(B&ZiEXt#~IdiT)k+@6x1z_MxsEt-?~n6C+mTJC^5C&LUya%FT?T zKNCOhf1y07^Qz^isavHlO;ot_uFJmyr))u8*I-ko-}wM+}ClxtFfas|?IHqE&>dzZk9 zju=~0D*s>hSa1>e$ZO_*>Mq z0-mmExs#)QR*74kR6j%Z?l-z%SL_#BNfV@B#Zf|QGQQ60HH{Kxv{pfV;l4&*^vTQs z^`=&seQ`8goX5qQa!I*|aT31ErBvs*>%+;Lll3{EU?#zsyw>|lJEt8W^Rl%#6yDS7 zdS`8iccFS;P1N3EV?6+&noSCp-s202k>*(>M{Y+uzr%hF{Kxr~pfY6ffKmZR;X5oB z(mY%7?D65}a=Q!sugIIihx2a@pJ{LV-Fau-nX;$nU+wYW&W{M~qqA2;%i_^x|5tc; z#PiTMc8@qu`>HibU-a{T-(P*M@@d|OC(+&G%cZ&Wr+$@kyvfxwZ;xEnLXP?kQewq2 zhT)Cyy!JTVS2IU@DyIKTK9Te`xkctqt&MfAeWm?R+g{5C<+yQ2d*StWr@)5%g1Tq@ zJq_H$sKD6RC?!1+zKFv)k<}+3?!2*0^T+QrR!o+<%bf*-ig(4l=csxZ{%Xl1A^tat83`6d<$D)=B`Oh|;b_SwJ-%T9JaJN4Smmu)=< z9AhFzm1$J5dYLYH&xQ_Gs<^wkb9l3RmZlwwmt$*xKmBFh*KR)+#CJ~1DcuVmnX7Z& z{~}K13=cf1bQ9|7iJmj=z2GBism;ApJV64~tm@}E<*DhbwkzxqA1HI} zR0VbHve&m2wQje~_1oazG2mr@5_sMJreB)ls(q-#aNKns57xr=M-(g^QS3{>zjAN) z-y5^+LEDSJUi|$=_~W`UBh($v)_FUY=vsPIzK>z6Z6~#SE{7|-Cn6+xkWMA&%8Yx;kgB$rzjdDdVMUDV3Oa>E(&jtRvUvj9f!LYN=zDt*tCm z$lF+{ZF4_zc`{Sod%RZdv3W)MpafXvSq55Dz^V7L7qqvv2ix5?-TDKZ-fCMLdt+O3 z`#Q%4|9_l?0@JhA4S$)dTt2OkTHs}#l>tvv);^nm>A;@L?2iZ07rx($U6?*p-0z&9twF9oat+JAJE*+h zZ26En95%d7u#d-jTd3_lrLn|*%jl5#uS-!68^^`P)~WXH_OjO2@(pq>XM!Caqt_s( zq7Ph?>H1UcvwEDhQ_9#T=2NyRY04?M=o?GL%x&%ush?5~rj~H2-frR)Wws^VYARQy zXU0C5dl$L3cr@=%GP~~SZ;a0%7$1Y@8f4}&4}-UTq30l1^QSsUJEG?_BZc>3u+qyK zY1?f3Y+Y?hlha^ht)(1N{-=DCtI3V!mr4z54LHu5_^)^N3QP;y9n=r5?7JZy!Up7A z88JCxO3r!C3mH#e%)d6`>bx7qy;Gl#xSIP1MtmzBRAG3r>3Nq07gPFpP4|V&>!}se zuB6UO{`j*?Y;??>n8yj5-Ayc;11e>E5q>Xw{?K{OeHL9`?g~ynkzU7jk_sP(VXbw$ z2gBtP;Eqt;_!iq(n!_W%!12sBR#_m7&?D7w^$$-~ug9BH^V1y0MIup;VL+R&v?C`) zRZ7YOq{>8GryEv1P@C+vc&~dpds}$7dBVJ#-M8IGnVW1{7r`dQ%1M@GmVe}4;&x#V zEPPwl5AKQXLoPQwR##v|8KJt3>e2(tR@-yiQri(rhWr`S`6XkmK2xt{EHYb)CTwMb zTmufV5%{ROfq_3F5rMQ0uqE0m+MC$x+V|UDQt7a!)rEby);~VrTu|4LSz&o{{F1X* zuDEPbYRsoP&o)0R^rHWpYjFo$b~!5ONv?(YcIO-(^2&OHI7+CQPoL+uxL0NdrtV7G zm9XxoC1!F$1y`K$tF@Z{vfx8O+X5y!N81<6gS83Xj^38)c@VQPWG6U`Bvn@Td6#)t zYWIy0&TH3g)JsvWNgK^IMj`#78m?V1ZgbjyOwRrhu*&g7A&TPXNH^LL3oI|X@ef?U z)0WR@r+?7D8_URvFDG0hm--I5bo2EM+GuTuy4V}9eNp#oU-dlX==X=^GfMkivuMS< zv1%>Vr8=~&`U8X59bU3%_;MQ)C+`hI%sfuZb)^lWMN9)#v=q<8OYn-OUc;C{9{v$= zs+1y)k$+QefmZKt`9WRKUCMHKJl@_(Wa@7O6aCzrE54JrSccjl8FOs+pXI#hbUM5H z-}kHU_uBE*e#iF2vR)|Q-Vxs;TL1nwIx_yx^t$>>rK|H^$ncQ6LDT(4D(%QgP}NbM zipbpI%xM{6=^NAgq$i};b2;>-QXZ@3nD0N$-*9}i9kgh$`OT#^_;s}k)vk7Eie3)> z!NSIKV?J4_wW;G;6GUC4v{wujF8EHdTDbOJ9j{i@zH2p%;ds(IfkmwdJ~j$Oc7pIi z*o|khn0O2vPF-^aOr%Toy5RXLz_EIT{KLl5V%Rv2!-IKLm_}Zp)7%Lnx<9p+1R^5+ zwHViM*ER1!ElH~;ev|#DanqhT7>yc|00(jC*7pZ-w(P4v$7_s$vfpGw_5p7y9GnI%RYt9yOed<5g0q zd_yj)ERxdxn}Lsmcyzh~iIvL0f8_zJbTHG*9abn@AVs zh9%un!1~qVw6uU7sS^2p9kt7>{!gh@(^qFSb~o25QDdgLHQx4%ZHgtQ+*~MOwA4pw z-PB6zCN+}kt9i8{T8ugauId=W3$t6CSXXL~H}EU`88Z29@nqj58o}mIBR^<^YUBbu z;K zGXx~ZF=VucUXNWcKpmyF*JiMXcL1aDguH}%gdB|mW|fC z)*#z_>jY~fYeVZ1>tyR1>q=`r>my5F(Az`h1(G4Op+bq|T9{EIJv3dFuujnE}?6SN{o+f2u$0!QV zYnb{gK$y!ZlC|hlyg(U=*hkSuGEeq8C2+mmqQb zfSz!H-&_Sk{2O)8^5Y?E0Uoml*pcqMt^w)=MjkwB1wiDC0twIt-)VW6g|>hITZ5Ij zo_~F;u4Rc0J%>56p*f96?^*bEI^+2qLsroSvTOW=qF^G(;Rb(j2Bgh+V#ZUbUK&OI zR3UH@xoKr*@EngsFMg}~u*y21+(VRVDtsy6YTg=Bs%BZFcFC*||B z=peq%|A9h{H~hd_%qA{ggm?c2N9_QU>?EsqE!eZ1U<9tPHumC8+yK(z6H(7H$X9V9 z+R;Rgy~N)05lJn^?^RG^YguD6K#C}g)@(eC+rWwRVq8DrOPvA=qd#NQ79`GsEUd#m zBEWZu1Fz<%H<*v#L2ZoVY2UJ9mft|E+`+zF0~)nEXv_|9IgA5UasYhlEYJpfY1tcE z;6t|+B37>B)f7NjeCNGX;>bRXVqTC$C5S5Lp}p5w5k91wmo-w2p81%r1yD-wxw0bT z_8YC83eThugIJy@bUZQb-K?ogv?q}mZ(H!+htL^M;mX=Z-`Wr_e#3l@0|VL&l*wS` z^(NyJM0C0i$gWXb(G<*{F9MyP82T_`(6!k|e?c48qy=SZgPpejAnKlt$hI#&T}7A& z8h8a+Fx!~zX0$yo^Z9`LUSzDmU2*5OtnkUSwm#DFiCFt*qSEhZp^JV!ChG1(>%sO( zyI--wJ+OSC3Av9I3}iI(<-@B9pb;K|o7oM%XFE}u)1V5{xq}bcB7z0^h%f&Hb9t0` zn#a915xGCk=tVOl*FhJY=DMA{w;kNqV&-Kyyv`FDht^<%dxE*cAh-Py?OjB06tpTz&tgahK1G-4$Bu>%l4Jayb#vZH(q6W?jIy z?%+-qA{X!I(NVs25uDLUK0n8GK5SwfZSo^$=?;j72Owm&^63sx3&*+QD^eE6?D>9t z(Si@4HgbacD9-$sqix-oqu=pK?L?-wXU*6g@cV=DYc~NOd<}x(6rb#-Uw65i4}Y{;s@>qI{NNgLkKE+2Y2 z1&PhgIQtO8J}LMGDJX)Z#32DTFhX9U;>jRvoIEd-JB9JYVyuM{e76uIUXayTH4EEP zmHsv4KGpfVOctu7GT-&#lXA1ha&gx{{$Byqiw{v;iC5*AzvisZnn+|Nq_8Y)R2f|l zBVeV!p**P;D|8HVU7pWN(91wZ{ykE34$bwDR{GE{ujzxE)h58*^pIJ6&bYr|W}dM& z*RW<5u||#{l^2<@N6hC#WbF(q>OM1fi8;N?+<)ZX*Svp|E4|#+;NN5-*_j|wLTQ)4 zXt# z3O#?vTwP=p_%OyV`Rg7U;smqt9360i+1bV%EyH%0h5nk&zjM)K)3FwOXvpRKe-eAc zdj1;6t5xi6Guh);GNTLl)&}&^VRo{=8EId}!yV?=hqQmq+`eKwstfep`(foS(OGOSudg#NW<)!Dv8LKjkUQWg_ zfN{3d%Wx2Mfqci0kt;^e3ox=_$VEj)usUN|7J0A0Up@(L#O!p!j_e8!aUgoFH`yTL z*e`p62b7z}2jFE-6M{_e(idU1!bperY^ zBaYyz3H;rI8Ee9Qx-x$adEJip`!Kt0xbtMLn?ek_3;*xPH%9Vrd%n|`>qhgs1*@?q z?U;|vD2g75MGj?<8EdeC zKXZ>>NLDlEWDv9IJ5e-7g1*zY7sQr-1-YHZ`0l}q?9IHKW+lzT`i-Jj0ci8QjFa!* zX!eXVNcj$|h51+oD_J9J>FrJQ&1Zgn=a?a&m6r4AK4d$IeRv(a@F#lvfFIu(CW>di zrR`-|$M1P{lacD5Wrug;tkDV9nbxehye$T9x%US~y%eJtgf-HVK5s#umtw!Q;mYgS z{|C{lpBRk@_KJZ>WE{DEcllN$o}I|Od^Kj4GGDP-*-%x`MRky|evD{u^p(VzYs~!- zbjD%UbWS9>5o>Bb^@*qP+D2xel@Sa@wsSL&;mAWB_N>?JiXD;t(`1CfWrZXcf$9Gl z-)+y88KAfCW#J`Pv+n*SyR9AwjU!Z$9!cEf0qblgw&w;A`iHr4F7w-k)xM3~!voy! z7%O5Y_i4vE?#pWOXUw-^`)))tB_kgPkJ-}i2|eQ5qhw}1m9~HIl17G&VM0U|g9(1hNWVywq8Z*!R6WHNIE_%|*y ze;t^w>PS^?)^!Ee*G(iLniWx)XU}AR4n$^jEQa;iA3p8mvmHH*jUOxDr~l3|2TNId zmC=DySxw_Ocm2)2;LGR(JIwCu%Xb({t3T2Ib&UB<`ri%dZjW3PV*V>2`|Z#|li44_ z8QEm!HIr<~+I+H-Hf*AWlekku_J!iyM~AyaLl0SKT?6)oNnAOU)m4p=k3hx)(WQgv zc`&2Yol)wGJdWbmw{wW-!pHP42`zpRiTFlFoG%lw4EOToWfn)9{K&G4-m}6l(UU{W z%QO0$nUw`ph;M&mgf<}~vCLN#tN#w;;mfsv8w6Q(vF3x=ky|p_8`#s&kxhG&K8!|- zE~IwFAFQfFtgVN5eYf%RfW7YlzA{ZHj4!4x=){}Y2@}~j2cVY+GJf4@bzU@XcgEl- zSMJ~*UC}2FR(JsVp&t87IrhST(2cv$gwvTlE9?3^YvLayZV56cF?v<$=dVagdFI2H zD;G?575tyP&dZp70gLX-ek_D-))VW*WTs`-QWP`E&FFP>mdqAG2UKFk*W}qnnMDWu za#L9!Cz0u5jJ%0u8i^&Lv1j_uVyUdqPOOk^Sf%Y*w`*Ao0#fd?2Fmm431sRH%LrVK>)B6PT>x_t>uj{70wFddQ*g#55lL<~w7wM@qsNiSEct zYes1x@->K6(i~klkr^C=WHo1}x{Osfj#cH$R(M2verI=0M}L)MwhABtzFgD^%<=E& zK;QW#4oO@2`*x0HQ-IOB8_?ePct<1w1XhU^D2GhMNAsv}EvkhIU}zf+7sB5{H$ zjJAdI(*^o*pFN@kcaB2i&SU5FW%Dm$4_(P>Z^_kFxjH{*VmES76#1x18_UpYUxu%p zUY=k?641Esv+})r(!vk4?QitZDMsGs8AxQDx3JG1prt-v%6?{H8)N9>4(`KW*Z4il zJOm&+jnFWId8#kJ%6CfWz~A4vlZU%I@DZJ1PSaq@t&AR8g(cS;Eu*uCe&dP0e7Gv? zEd$X2v1sgTJa0N0;tsQ43kgl+z5gTWD!`k_w($5&JgMUD?(XioK#}6^?(VX%z~a8h z;_fWYVvD=9rCv%cP14xt`={^A*EWsJ+s9%l@k?-?e#$;3Uw6-eJsJm{ zk%_w}Lq|@ACIp@_Dl?Dcq@No*tkakcaD8$2P*|~W%;GTqUkD$g5PDkuhEeEeiyA{q zmxTVg2z~&T0%lYMTCFK`S}J&cHqJ3m$7=*=rJrt_3QkDF6-MMsjeg?&0=}o89t~jc zO(N2mg;hTfj_(6Hy~CZ$V+0GZO1e$c=TZkeQ3W&8V@FlMdjxt!eb5epLMI`eLqU^b zn6vH`{6F5+8hqUty4r*Yrwr`qqoBxWjBg}X?IiLRpJBC5L0SfY%ObI=dq4sGT+#sS zk$QngJ7a#4;M=X>gkm_^?#G@d3-b2{d4jj_4Y%PslVK_Pg1VLPd=IWa5YfjFAVm_; zeLLYnp2EIvKlEK4a8_A7-vepX_fbu8->dk%2{bv1SvCgMzkqM5V)e3Ng`Z<(c=$pY zkkf;R(a5lGMRC2#pdtlXbwK)h;ad8c?#j5vW_;=nR@4Q*YC8Dh0A9TV>b8f@eU9s0 z#rmy>B+6}CZq4>^z~ z`U&awLe~BP4>X4@8;8}Zi|cg&kMnr<1te}Xc5xqoyZHfYqo45>Kpp*L(%u1F|6BCiVvx-$2e`8N8^Q(5X-IakK>34?V6?7k>UFXc-oI zMBn|!LsL%%Uw6aq>H=oD9&{NFy`;yk;_zM;c(DXNUl$x%5Fmi&5z3xx0h6_Jh;jgKl2PZ8T&o3NmmME0YQ;c|eoz(3WGN-`9gWx3EsX zFt18F&ItCsE1_3D;0_^(LtlsP+6q~^i8=4e_w7=lQ;$KOG9Vo~T1FYzl)*Sv+6$G) zi=d@mLh6&jO+7&2bXdxJ(1g<<)v2KWR$N`z&-zJ20&^;g$iE)#eTLaD!RG?tnXy>c zzVMe^I0f1Oe%uVusW497^C(OSpcx(`4Kte46iYy0B z*1;OT!5+U8c(ocVt$wyJ9Gn}#Ja*!)dAK5h*Gq$|UEnYm{E}V7ai9dY1RYyqRX0QL z#KQtsg2b1G|NR`1iF=T`ir~sCuvRBP|7Q4)1IpYb%(M&muq;ks{{v(aJubuI_akVO z9vGp19?gw$rsFpU?$-CFkFZNx4!ZryUxD3NPyL)-bMW)OppJfut{h~KB~IenDp1A; zfe%N(Zz>56Sq+;z5o^;0biM>X=MKiQ0TMnP{Hh|`yBQ9C0o@&* zuPr)57KXxRHUX9N-k}i~_Xp7U6!;>5oq;}+n$WHTVGUD2voMUfCH7S`{24na5C+c8 zfQ;Nje0l)Zw>5MWf$aM;>=?@7yXC=m`dLyvl3WY#=yAw8SSb;Hp}r^AS35zg0lMCT zEcj39oN5ypN&KpgJx`W$;lC=)@=R0`_2~wqvKi5Rbi}=sQH4R)Fj3 zW8KHViwCGT@Y)W3%N*w^)%qj<8#kb=%;k`9okFig^T%8m5sXn&G6{TVJ=mm1ujC%6v2!PkPZ%O z@enrg2WS_8Pw3~Tt3cE0Xe@dpzdF{z1%8jfoSgVu8=uI6o@)Ur>-ZogKp`4-t}STY z1oLc;XQ8mNPa$J8=$#BbdH|MwBea2@75WIi2}92C80fPBUdnHX_C3ON^>f*2(07T@ zM@ula2p~EhfX~u5f9IBh5lXQBpF!X2&~{5e%URHvOW_~fhSi!2{eB$R{ukaz9`3B* z_3`j*&VyS|LBCuEy&vEnG;s#&lmjhw1)p%?q33(_v+^O3k?N3-w^;v7tcEW0pYiR} z&`$^PS-odn7G%heYp8g38yZx?C*61`xTB8jaTwov2Y=@z*7qhxrNbC188WXw3xO8&0Y8GkJ33BA9rz;Uz>AsS)Q4Ej5XjPW z*uO!bP6J3;6s&a-aL`n6(^hEfj*y&kuzJbJA@qfBu^5z(2KRLZ7gq+&D}vu9LszZG zJ7Zx*DfBkd`LrD*c`~$EIp~KE7^R-Oc!9ME;7WQv`9;guMDmh0`Av$cV0ZRL1rarMjO`QJM5$iO;Q=Y zzy!>qH?G+dyj1|))dBk90-~c^@OnRJO$yv-hTdHZpXnm>>^h8C&sucC_%`EnuV4rM z!gto;?-FQ|RhVNJ%nc_=vE%Co3M-hCo(a(Rj_vVHJ!|E_tP4U0!@;|yG0%!v^=9x1 z%7PE|v&N-hRr0`FK5)bZaO4eeSvpq54t~%hwKhnmE^Ti>FCCFc&z4oeI_lQFAg-aG z6EBE)>$^k0)*J7(#B6o!o~pQ45v+-h5s(fpci@`UFq`7gEqY!v6g#Q^AWaFd@cP;P z0vJa+_^$w@tuLlk6i8Ga67-W#xDt)MaNxPDd4q+-7Aua31Hg4yU* z7J9_47_7d2N?p&bRs}!xfQD*;yLZBRl*B8I@UGs?G6ysH3Hg!p{k|`dQ#WR8zk-ndr&?&P}mjM)O_+-CB>uiKSbOcwM ziOBpA{5Kv~)pu?O;L}ZlkF^H(84H^<06xiLe0LqRgpT)f7$e>cFKswt@j5b3HvWGJ z{dOB8{esm|F`F>xXb~DA8>=AV6*DBXdA`-kz*rLT2|v8A2H;>lo?0F}-T)la1)SFd zYuF65)$1H|zo#hpQ(p_6EKbX27h_?C_v7J<7);wshgZY{`K1Z0H==U2cubpKT6 z#j?1Q9=+=gZ%0S*=!?&f&cFB6d=1wh8e$rvP>bLJE`+x=5bw^wT|2@0R)ZGNYg}|h zj|j-Q8NXHNFBTlE*LIY|_}YV?i{YNPpyRXg4u^5tuvU6}SA}oc2>jn1GYp0N1@LMj zxJiIcwqS)gJW}xr&?oT;1{_6Vc6yzF1bHY232O|=st-NW23BweH1t5sWIRSW7GocV zd9B2_`{2Ev`5Jg5e8E+CXC`gRnqGZSN(fY+Ddy36p1ffz|=+@%iIN#FAq0uPxW zyAJ3p-Rk(Dojl-DU7vjb-@bu&{~1>M1FYg!@cAnEnA^emf8uc;@}&El_wqmYBR?wX zfn@qY3mW<|3p&ROUe;?U(()-}08f<%ujutHdR;^n@S2`g(4#VX#$3n8@`8(v`P}v! zcu2=^s|KEngf7&v;llCRg5XBoe-6w4rUWgbX9z(`?6LId+iUQ(9v{^+XOEz_o@4A; zn3)HUi2T{=H5K9Dq{t@w*tV)B-Er7<8=xo+^ij{@sDl>eWEQhS0oy@mU=O zYZ&g+9oMS}TFCJ13SzDX$hE!>InZCa6)}UO!|**FBgYMm!C}5S>Wv$*EFLyh-`TR@ zcrR92&lLUxJGT}3Z4)Rq4!+A$Xtt@)kb6MA`G{cd#?J)kC>?!qCanER_+ay)<5t5{ zS_Iv<4Vrf?cCp8y`;X>(et#iGw-c1#4BfZ{pWBQ7wtx>-K_l;jraX?vMQFTt;FYIX ze?3R3XMgpYt#=re9~vhS+Rh2j#Ev-xFpErREGuSm8#-Or?4R*W_vl`LZtq}op5fU+ zJg!1#U(UbJ4UAwG^z2zgr4B<7BB+7>)$tbGYcLu>rjW6=%4ox1idolpA~pi~{uswu4R6i}-# zWNIpG_5kpKp7HMsYU{OQ%c~gwIZ)*a_654tIR{@`_k53F*4ID-J&$t_bGnTg#NoSdu)g0w`vA0LIq*wS zj9QPq_5ru*QLzEwv>}lFD9HY9XsEr2)oz1-c?LVWqljo6Mi$@!{yxCIXCHC_YvFM$ zf*-OLI%^!X#Bj{MGdS}9RY7`gLBxIE;=Vd<^?Izkpu|BuyM@u;0ymrnj~xKbj)C)@ zVyw3?4%=;2JP^aZ*{QeG|atl|~GufZPwR$AFLcWv%jT=(0`!c#c zsR=3+!M>*eWMB94YU7>Cn5|ylR2Wp43Tn*6e0$@$p7m>nIY)se^`JYOVm-P;yN-Zf z)H7HUKtElVPsm@VZn%~n8)}^Y8J$LENLwLXB|2Y2c7XMC!_vHg9oMad4`Y3TnIFZ> z4uN`qf#YvrHhbV3jDS4pxSyNw-|vvBRj~g{As<)qIG*n{?8~R^aje7}&{0R0(!15@ zdjuUjC=9Z%``@PgicA~kRtrDf@G}Aw9R}*m#yoY)um*FB0>90~D?{*_j*Hj|>lFd5 z>3}q)gD3JZR=px|ANXh&#-dC7Huy&UkfCm^)k3DZDXKK;;J@aGIdwvQx+S7!{SkTU zju=Zv{O$qWFc{xkgPF~P#BYaR@)_EPfHk#2XX{$1C?vT&Xi^*ct_`$k7mRZtY|TJe z$SI)5O4y+l;HvfDzo~ewC1j&5v}aXFhHh(h+Uon2Nc{C7f}z(%HH0MkusZJ`3s11N zdNsy7@cIepqn#MTA@Gbov-?=$MP1)hfuXGE; zY$sy>(H$e52QG@jv;L5Mbftu5xQU-H$ksL{o1#9ZDt_AHTx35yi$?##v!st?sYoCW zM^a|0FonKa)MoT!x=XE~dILeY2+qR|1H$`EsufO2OhsRY_NbM=Po75?(&gxbcm%7p zNDI>rsD8r4gtR6h61pE5 z02^w4G7$gw5e?DlxefJ>3dfnLUbIZ5Q8KW+AJK)Gxj3n^fr(-YGiV;q3}b9Kf17*DZQwSuSIOU{#(uwdsox&F7-Xb4@piC| zuZMfPE7|iXP#^fz&4B#fUoI;P@)fzJ>Ln`B2kE)=JeMG+}enkhFze}ZL7m~sHf&xQUj#}tU^D;z5Bw3e^i@dmb+2yQk3kEn8Y*K zl405-;37;?Issw-iZl_33Y7v`{w;y&;s|LV@Q<28;{Q_H0_X9ywu;<<(~P^(CF&J0 zwOU~PYN5Kn7`hKMqH3Uz(I@gI(4mV`kjHFhLho1KOJz~`BJ>=gd0aKbRxm~O0N znqV4dI%wK#>TI53jxaYhx3N64t#wogjSng5h_t?A7X=DsTuP1j#pQf-UIGI9AzuN{ z(aZ@yJ7=!;{;f4<2C=316fS{_rm%}sCTj#efzDwXvQ3$WWH0rPU_0Mmc{lwIsj>DP zUBuwC%9q5NVnOMSJVtv(j$;NhQB(#HFRH7%FzW%RV;T)^_5h*mp>jwnE{?%D@X0Ft z3m~1p1-g9@9Jo!j68nLFKaV1SM0xAeY10zaLstvJkpXl&6YOd(j2tJUaK>1VY0OO2Mg}sbn3X zUhb!huyeRF!XZPJp|~l=Sk`1WoicFdE9Q>21$NCo*zwrb-#XZG5)Bnkg+@evi-@xH zRW77Ezx%$viK~-%B)y1R6kmw7{Ds|CcSq?G)qq=IIAeL_nBqvV#PU6fSyDlN<-8bg z6)6(v-zNbmo9f%@8kpP2(@}JihqwlOSz!t{n5#m!)+(!w<%>W@tw}GU9|4K2lX%rj zd9B`szR}`Mg#>ENKV%g09uKvFKn#DR&L+z;l{t#}uJj9L2cG&)d*Ayg|M}qCU@>WE zphMuQv{`KHDbQ&)6wDV~saYFqbr&EN{&TmLt|} z4lZ=3eTem!dA+5*^-tS&b2^osTQxrZ>y@vY* zf%GQ$H1Ns)hi{a>vMVk@?tM^PMQlWm_gDV z@r^u6Gto7;I>L2fFHWz;P{TE|qTnnmIu@z}iEm^YHH9h1p5+pR@urWKLAG&@4k4i- zqiyA_%gn5iH`;|6+yg$rw9UE~SO#Zoe_7|+%2>PDlN{e7UOPrIKIb1l;y!KqI54JZ zeD2Tc?&_+GjuyV~?}X;;GfrfUY%9Y(VY2ZQyOmrLDCByQSt0Xn`juZ>vx|9Vh;Ouy z^epxiyO3+kmq7=TD#CkVf$%Taj22bDzl*1WYf-Mtb<+JH?~;F;__wx>s(}+#CmD3D z;41J2VK~2@ZAQ1m`RYP}qrNb#jmcZzyV&dY49$DsE$6=`hO2%mjY|{O7=H2V=|^h1 zINVp()7kx>C&}BvpBX$Y&6P{yeA6t#L9J&7bJK<0h8u=`{5!g;wq9)FxBIFG+Dg4N zD^9Md>@NPXP{P>BRNBhhF59Mu%yR^7t1KJLuMDLH!O-7W(6r4s*Vx4Tr}eq5Sg6Nw zKBRGY*$5?kLf9PBL-9xIoA;&OHh;V3W6K|J(#r)>sn3>uVMD{Gg*eP3>BYbgyGCCm zT5AK9LE_UqKBr~Ilb_3eMyC(UT;aMdo}$+q7n>bcpZT0Q#TX&X2j!m-wX~P=;y@v< z#U17LUfz|ya{hPTrg>Se zGr6Ta(Y~J&Lq&7FjaN-~O;Zg=*uS*DgN3}&o)I2LUaYsZzj%NN-Va7fw-uMxk{-qN z5e6I18`cWVxHRfFqP6l_ydG>6oCPG4eneMVWk2&}4GoO*&8I9e)-%=^%MR;aYYXc{ zpd47NLo5NyURy=mW6MyJ*)Wn{W$b1dV!dju9NNA>T!Bv^bLr`sOTTUXbn-*i$3J6c zrcmypL@CSr@b!_eBc9t1a~rAIS_{P`E(rz#;{$WO7hOAY`#E1^&dAu4Gs}BUil&we z!_8Ce*3eCka+a^$K~j-k2R;T)2Hbh|-Id+V^M-p1`!@SZ_^IGXqm+3z@}gxqv|w#~L&HOHCzTPj&Itq<&7 zLzjfj4*8oK;whPO_*;jsqrN=+9G)1R^;}$IXki~2ekG!6*n7(owh6gdX&s#7Tk3t_ z>+j>dzvcN{4(G|NnCwfr$Gny0b>tELk>z%%C9JN!rMVh+jQCwn_OJG2=XP+OcE-CB zyifh(1Dk^;blh96EFq>-2Bs0u7&aJZm{tl-s-f0Q-XARM=kw;cemWDJ&2ziB8DA0c zyedstFI`)xb4I54H2Sx#B1nunObT1Z=6 z+g000+j(n8>nH2)*2mW8_PDUmVQ=lDxTo$`sUKrYe9iiL`fKBa2U#P7SNW%w1V^>7 zOdD&;WL>H^*x!5B-OD2c3I>P!W_qi89(hvTlxvH-o#&yy3a}kJ8D`?nX^wbXar1f3 zsjdv(^&WK9bJoat1&oAoz7D}j(kA(ux*By5>wpbepIOV57gG4GTzj@9vz~k;X9oWB zG<5FG9G#79%#5|9B)kH4eS#1RJ|TB`Ih9>@Y=n#{QZNg#J(~uM=Sl+ z6~s~a#gXhpZYbYO$m8PKqx2r)t&%0JlMYHBWmPLm|Kv&-n^@jkX4+QUs#-%WTg)#^ zk4(v?Dwg@yIrfJkpF-=0-42=W_}kXg9AX)4Hko_cnmY!CwhO5*#Q0{UcTVUNbKy&; zFCjn5|2pZr!<0Awa14%oK7*~>L9uRHuv*|#(>Ebz-eIq#Q;cNg}I z2#l0w)8`F$%u5`b9A6yuEzRgm<&eLYyMnV&cCYNVu0B2samfb>Lj<5$I_Z;PHX=bBw5Z{L{Z#Zl&W547G z8oz5z^PXpnPR)&f5t|*~EZyU&qK;y^n708FV~-=;ScR+|wB+r~Et{L|RM9c$ymuVV zS*MHP;+eojf11zht>MoRgT!bq#Pr3o+WM#EqTx8xT7BSO?D~{FE~|9*rrcGYkN#9y{I8+ibNqv~99=vMsclt&C-yku~ix4K$Co_O=amTydOoytc12 z|8D+b+F%-BK4^)zUa(5W>CEn6QRm2C3zF}}_e|tcXJw7|ewVH@4~$jp-9pCONaJmK zw_H46g5Lelc><$fs#aVDT+TQ@`iUaRk(^|*)h-?v@@4Wu&QR+WQcT-xXKh0Uko&L3c}lcAbl+~*zTNy^=pyU7)mw=6J9W{4)tWd5WOVK~C4Geww(_|zZDUb!9i z7lq_#@s$6lx4gHtH`RO8e=pcunWXX5Lb@R^p<2^UVz5$3jPPI1E1I|3bJ_C+_y=zm*_-lEBQjNN>%r()3J-5M-HJLFdA3cU^cxsoiWRn&6a4( z7)xcW%L{8UYX>W3X^xdqO(V_Q%@@oMEiG(mwqcIyp=9WMhusp*u99!N@-l3xbrNSL zc1kIiaW}V!f3X(LU$QK5^s_rGX~GbCw6e+Pa2?Azl=UPh#<|$d`W^;bD=oEo>JhoM z)C%6NRk|T1$pf`7^ar*kAIlfw%dpAheYsR{uy?(ufqRSlr-$?}lNfC>V;52kbxp;M z8HOJGQF;UsCl8a31uF;I__lgw;O1=$64F5>9GJ;}QJcx0WF+ctJEBM2L7)f51)2x0 z`M3DL`gizS1$&E+C0^~URUyaG3z#izfEC#fYzXV2JlF->ffvA|L)j0tJ9-!%r>AkE zU@%=Xud~ds7PgMC>^2WJ4K*z_RyTey>^H16Rx_%`$)*aXGNv)++oo2Qxt3Gbe{Jn; zZ>?Tq4YsmY+2_f9oz*BkIgLwSol)Fb&U+2JvIc@=>0x_iNilEZpO9ze(m>7YpPiMt zD(gW`0e3^+gJ5Cx1JQ}PP3^^O<`R-xM42d6mns9rqJdfrd6x%-hC1z0nkdf~vjbQC z)dNHPM*<{fHl4c0j9~Y0`?+`QL8d*qRcS0;38n*iC)4}Q7aFWBN1_XAfUE(G%9_*{ z;+uLCv6zj)@L-m|i$B5tGq@PtG&MOMID$)1eRPhf2`q@>)H13$-I8vA4t4j)EM$;s z0I#7eqQW!K!FLdy&E#;=LWW_Ov5={=F<>}t_*2*=3>SFe9`E8}x#j#@zLCHgEXFjW z-&oINHf=X2nhTii<}&7?rpCq%9J;2;e+Me%P0x+bUY-3%j^g_4wFk4*4RnxiYZ`{g z!gmADRVO>iTp-akG^a&&`<%$!>F%rE@4*vlI;c62xlViOFJx16Y@Q`93|0)x3)~3U zgVWF_bA&QQ4X9SN3i_j6k{ij(<&pAFWh5d*$5lP^TnM!?m(c@rJ-R#IRh-fVX}vUC zv8v^f^Bt#^A-e-b@^9@lu24qlp?s3xORuGBvPt<1y*znD-xd=?$Y*3M_RrhNljJuN z2t?TTE<{vY1LE#)sIJ*hbS1x_Ypg;Q0OHMKXvT+JKfXPGpX*&}QXR%MQ| zqu3E#DBpxX!X4pWaI-l(qEM|^2e*&oxZd0eVD@d})^aP^RZJn8C$rS|=*>9~xGjC8 zcrgH9%&+v&nxMZ|74%%KN_7E_?q=nclp6dHi1L3x{AHMbwBHvPCo)oJIY+63KJC>J zfBaV&q0Ea*vT{CkX%F0`V93q#FiJLj2T-+^0wu&ry#%u131%ttC-XNmoiQ_u5w&|o%>l+r z66r#R^Qm+eu#b*0er6_no!P{MF=y#xbQoQOo@?ywZM&XD65pqiccA=rm6K1nO=@Mn4Rdy*phreTtj{IXhg2p;9G8WDRP(Pk?k8q zj7B`^5omf8oHd@@iYm%VIDz;a^0^BXsgImbW3mGKF3E`W zS73Qw-;lGIL^w$! zm4kfCcT^3}M*Mgcy3G02FjU)i1fTqg7(9b+gIyq12gol(A#xf~hvZPDzYEdt!sISw zA16XC&G?K;ydv)a+2#XThU$-=+UKFwS|E>5oz%~?UjdfJLU2@VbWyAb{F9PuS><1a z1q#YdbjvX#K3E!&@l@h6*1k7&23^LxP+L&Z)C{K(hM}wWHtmyA68-rm;_;1GK>fmq z)&u>y8P$?%tj+{_dzw;ROOzieM-hGOt?owtVF3|IHbP#hCDuIyd4%igE}|&7unwY( ztC67?hzNEj#Wu%1QNv6=n?-c2N_pMsl1 z024%guU1r})z|2>*;q|b_o2VfRy9^#4lL^`$Yy*&AHenaL{mtnUm1$5!QY6MheEQi zqptH6kd2#>^T<$&KyM`{a#k_O06jn+wh=N0DiB>3pwDhH;>0v!pA*q#*#^9zNSyg= zhfcmXkgXYrEYM@jttce4FEy760z2mknTf0eDx`q?+yeQ4SL%M8U&zo>h;GzmG6vs1 zMz|5HSHa5yI%A`10q-^<`hc22EdiOBwV>KbjOG{c&_rb9ti&Yfq~EX-ebiCfQef4s z!}Dv}GjzuMj?WH9Mk57sQUa)LN#MN;ic={E8Go!6!WeZl8ok;sS+Bd$?yJA6uhklA ztTIK3R)WBmUaQtdrrbtMM-F5>vh&MPJ+c6qvrpg(Gd{TlbSKcIegt}$T7dvhlcUK& z8*aTRM=RL&#Uh)idDbRW5miqK!ka~)CpA@}t!ki_4pk;FV`Iu*8OFvf8R=NB5m z3h5mMcjBijc0=D$Cm2s;6W7T(sFYib%+CR6gUN{0c0y)fRLh`pybaL4b5Z@j993=Y zLD8P5Z}%d<*o3T#+SX<2C*;1|@Hr2w_fZ*r7+5_$kOwb`y#EyRUmgc9(L&tEdYwRK zV3S$_{RaeO66UHkAP-ZJrydBdjR(^Hb?Q&p3J3Wb=K_P;Vj!{Kfn0R~R$piEZ8_ln z`+ylf7dxD7z}s|@-GOTK6ga)*i2=}D{gCy`Q_Erf-k^dt5naBMaXPsl^0+DJyV@TJ z?N@-y(;6 zL_3RKl5fz-yCW*r-@@W58ft*Z!|-d+*O&~&DZMp7RT=~-8%lI1FT&cq!`f}s&mKdr zcO!$`8ucc-f#1FjdFI=or3c+&H=skrFXSKn(7(l~MDiecirPl~00R1MMDUH6VFvO^ zk;pSvM|A=LFXwkun2ZAM^b>U*5JyEdPQ9-Fp^ZfLF$5an7g3o!jrs@z74|ogYkIEY z+yS!9#gNxaL*@A@)R?b_v?d~N?S+m2s;Sx&JkbzY*AC=WpyMwh(@>$c9Mw(5i9>1% zuD=!iq;IJdvR5AUF!nYz$SvT44aj08qw?VdahJRX?C-(QL074N!RIqj8G9V~jqjif z(v@$>wq8JHa4oWe>nR&Lp4Z3s5+HLjPH5PWB|m~Z+fGR2KKOSl(0M+OtVW)}RZf8; z+d{T~0a3O)dV@@bhKWLs=nksgd!b&w1~h0JsxPz|4}R$adzY#Z>Ov(}@v0})tEiLM z3a)8OJ^|8I6Y?_9!y?H)am{L|2dV%MNJNdx6!;+rwN20g-HB0TePjo_!goo4{Xc;f zsDSl%tG5vqm;;+}0oJN4L6Y-v=G{atf!5rCksH-=>U+77YE(<0F2R8F^P|XgayiwL zoJ{KY7cT4w*Qm{ueexSQ6qLTM9)MsWOm@R@#& zxgU5yKe;gsaky5%-wy>M^B6>jI)djH;>>;!wj%-KJ%!Br4Die!IIsK=dZ;1!ldMgx zLr?OR=!QEQwl#cAm17&f9sX)&@6SpSIx+W@JNS(uO0%kHUPUf zOKAWUz-ww-^woW=TtS|_vw8^~2o|e3>R4pnE1+Iu46@ACp|fOkdADi_YJFtg`=egs z6EY9s=tsMbtPc#ilH?vl$(BPOGFXXtwJ|LE7TBz{KwRtt-`0b%#*huEsZ?3?{&rGp z=w{4E+Rl8Y_anyRpu3&`7?$Fc!rZ8G4Y4MN5s zPp&8~2>y#4QBS|g|I(}a{_$@KE(!i8-V^Uj1CcemC|yTfsel-;N=*bs>Jzw*8HzRug{3HR>zKKIrO)C!)JR!Y@j2fIK5TflR8Nlt_` zt;88LO0BCDP{zq`Wh4CZNh)gX&?%uWb%Nf8YKUe`H(;}#q9@Xi!Jk*?Q?wNo0QcA@ z+(W*Wa9`MK=wWzdC~j!`7$4;@3UDcz;_fq6cwuW#Oiyd?Ki&tPvmU$;PuU@a*@E~3KU zASfz~ng{PWlKKk&xt@MzSFNGAu#Fx8N2kPT;yO-!`8k7<*sv1zE;Yg%YJYVw#k^JueR zO|*`+KDYEW_cN5_s?tZLf`NLjQrTaBrKWvPosrr-eM9E&xdZZs23M-PsUz$k{2aa$ z|0h?F8A*m{W8@3cb}>RKfJzNUtqR}y2i2EuO%I@}p?A$#xw}{#(Y{TdU+&-BTU}O9 zLf#*~SN`q6rjl7X1#d*Jn;VR7@doIo04kMre7BpbhI$*j{12`@RT?3sOYfCw>K@HX zrc=$BM#y>1MufH!7hpQDrx~G`ZwA^}#b_~&Gp;pWH`X^j z2c5SW$C=ujx|#c%gO-yvhhvMqkmHWEq1Emtua9pS38Zhin(4q^Hs)WuJC|JVYO5S|PgYr2nOy>Nq7g zP|;s2uald}{UZ!J!fnqY^fqSomO^!j1clkI?}Y=d3dYC=CT=2v&9 zjeyO3RDLOS#nqFfZc2!HQL~VxsG0O>W)EAMeaVhzZ!*o9hNy-ZK&Mj6=--%UjEC*P zHx()v4jFO`&5b*ZgmHo~Pw*NxBg3)6xX<{}D4H_OS=JtQgX5xOy`8j-5LVDNG=HF8 zt|{YD>d2&}32OYv#Ad1AGrl@^`3ozrC==Jv5NFzNZg0HC-(ee4leLY?9B8maF-5wh zUIuFHb7)SLxrF+obb?n8OWy)5eU`ixuGOxq&fB?7+%cXt{)&P1!Krdfbt2B~+{77m zFRbbl?I8TM@9?O*Vy{|AZJ>lG8-WB`N!}yfN3_rjP48685W~o})LHr?lgPeioXj0& zH&dQTre08ks6JG8dNm!(yk#ByZC*oEzl3px@hKv8-Hc073o+8L*U%O{PF|ZDnm1T- ztcC53LXtu{hi2Ld%V)z4x~aM~V9h=LYkg|-WLx6j@fiu7QyXNA%Dv({jqXnK*$_iN z;}O$6!&-hSJB*q_0Fz!mES*9v(=gCZK%8L()1TwHPV8g~y_MBT;y3^8yqWG&u3^sl zxpQ)@?hBqJ-oFDWVvgJZ)qi7Pd)8AMP>(j26yYm2(jwGz%5Ax>d_YPQbHo+mHL_6Of{{8pN!L?HIozWlKgdu$I!^!%#w!rWpfLef#^<_iR#C~@Hb}>C25ff=XUVNz;h3{ zxAb^2pgax!>z(Ybnp-QUd-lBSl{rUpW%q%+MS(o=pmIz_6)aJJQqf)OD$ZKZgxznd zE-<%jT_@IVL%iRcZp*u2L&4-lR$Lm!4O zh;T*x6K1ltF|}fC#5;faoC!ZelD@@1iQN(ND7It5sI+NWTRi`WB(aHEAoMW3GG7G% z;TNt7a|=7?B;}ht7rT+Z==zt=ROeFpZidH(-}$;s64?Y<(X-xE*AHjGY**&dto_+k zZj7slx1&EqdaQgzrPvqrS~x>AMcn8qdIf578^x^DQi6!cPFEf(OO!EKi5f~jyjD-? zs9aYUXmf}$#d>y%9SI;|8wQ$L%M{Z_!&$Bla|!*eo1iz*JJgQ#BAYWe*qdAt zp_XBlp){XPuOLE{4na?z$@One*Q_I%7c+y|pK}vEZ+vs4m1;3~flKMJx@9K6XrGmB zavN#6_$|mm-s>V$u}1!mC_bg`M4zZ`sJXMCLNOefiQQ^%q6#^gs!0FETxKhCGTW0K z#-3$LF&F4=w4MHwN~MzN!Aw~;hq=o>=Wg-C`8IqGM+*=5wLB-B5%$62Z8F`nJh1Mz z_jb$({Sa0&vRP!e$kCy{%$dRzs=boze)j8G@|_t10oio~7RNhEt zbHxo!%s#Vhye>54wlWd)NW@u+AqrLrIgogCGjIxDz;~SC4*MImUd;=B_ttQ~&JnYo zWE9N|%^v8i=t=j!4z^Sd6BDp|N}y!y|4*wGl*3Xqyn7aPi1Wp_QV;n!u+sHT#}hOf z=;4PEna@KmR+jQ)R&A$!CdN>On2W&pf5Qd2mfUAH$P8op0QJ2YBIF`jh^82Vt%X{_ zsr)IyY}jcq8crJe74{-SDETH!03dRk^%!b~d9v0do}q=GIReUx0) zLJpxFTnqjVKSbCo^x~tL&O~Q)tmvQjHMc{~l`J~zdlv7k<92%=2XduxnuGd)C{#T} z^zW#><-y{!fXhGGUq8?kH7&H#L3L{_$T`$1sxCQ`=&hYn1L$mWQEDoeS5~2W;P2EM zpzasoCBCjOLwLyl>Wb zrc%g~Pqyx`6?J5Vln?6^ek~%mz}|wk0x@CNEa!z9WCi&@_syS%gpskmViLY3d}$uT zC-z7mf8Cuq@+@pDt|G0EmYk?kb4(30)J=sS4TaFLD_LuRU^j7yy z^VJX5mWrU?!69-b9m*`FJ5Y<6SG(=BlYs_bFa_2lbM>) zI3pp`n^V`l)fXMyrV!|4cY>)$zoQbhV(KR8_dvFruj*~Vwamgd5i4%Tb7_4Xl-h|n2f+roQAL`2?&bj)(xI<)=%pmAOAh=d)T*@->$^2PO6wODffxLp7sY*z)-}r)w0~$+)Nm} z>=@=ay#%qZ;Y4ZbG;@?YB%CyujKhrwh5Otcs*yT3sOIf*)yj#=9Fnm^Qos) zKo!eq$H8 zxUaDDxjX!H;g*3k4l#wBXIU!Sw%ZhY<&dkP9@MZ!MpTTL6wxm7PQ>)^Nc(DGGu>Ai z=YNs)Df!8d4&R%8Pxv-6=4V{rWJA^{PgJlB5yx~gTFmV&V=b3Vg9U~yOvh255t)Cc zMNnbvC+-B|3VDW429nOd6n7) z{NiS;d=XUky%Fy7$GA+UBvYGOjH>!ih)1U*=cZ}riAQ84okwReA>3l_8y{jAWK1wF zF-SI;e+-6-p4W?a@I=L+{PUqNvrdbFlfOVD+bqaEn_v{HH-xa@1; zotjrB?-nXV_K7jj0Y%Bq^bd5+*-L*U_4-Axd`D^`wh;xXpxjYCMAV{_ng6(v!eoOc zB=EUh3VVrpLyaLD6FoFmGa~bt4n&!oq?@`(D@<$l1V{6Eyu(lwCn1)if80dd2YdYx zE;J$ZY1p^$ci}z5>xC~3{~Y#5=r{XYV-(X@JsB*YcQXBGk`^~4Zdgo{@5f`;CGbDn zr>}wdy(QE>pMIA=Jl?lWQ`V8BNpUwBx)c%4eAnogel7Tx%sHJeqelOdS?FDqFJ}v#@cT<#)J@|r$RzQ z&xCFby^Eia&|4uw2;&%P+|9(P8^w=#MKjJMZ~gHoc63bm_s+4t_-kopa~gYFN%M#; z+(hF*bm=^9jxhb;Z?G$vPl#R5L2Ty=RfM&nKU7CyzmUxz=SI;Tv?Eeae`ina+>6;S zvx;Wb$Ufve?;ht%3iOmGq5tm(tOidbZ$|#5K9@%Y6Hx`x)tBiztcdtQ+ zW)7xGQCrADWMA!% z=>CugjSHosH`g*E@!eSr@l5VPbk?=VspqxieW9a&p=H&9cIL z&Nz+V!sgMRsWW6vAZ(2M~8u8o=> zFEK=mlQ&46B^Rv7VRY0JwMc3?eUItG{$kg11-Mp-nr*>onj%9TjSS!qwU72YasivE zEvU>{%68-4@vI@+@WR;DT+Ff!HHk%RF54D+U&mC(e~6`|*pE6=9OXk|LNrwVp42kY}EG#-=@Z{Ojw-GINiS9iW{&HD~)f=qb&DKJq%aT*)fq? ziTuocZ9G{3y~hUeg75)e?0LQxw;MgAD~Pjw_uUK3mlPZ%UsQmAN`9b6)1$Cz_mJ^kuMSjt$}8kw z*ohQS&!Eq9Q`AbVWhwN3Imn%5E3xnB57ZH|4|2Mt(Mhiw@-K13a-6xkg^oKaTc0l` zgkjft&a@Hh(aCzpTFD+_FYNefCm|nm9Yjd1V@ZhSs1*_wGS9x*dYx}Y*Oh1ZU%C4I zj7^M-ogNn*!^X6UJD9lnS0!g{{{{ILS(86#U~yt5#kfV-%wDHAK>Aw{#gR>2h8o^^ z+yuUmP+u_e9yW@qq)rw;`Rc4EcM4@*Ov-PYNl=5x&q<&7H*(Y%X^U~WM@d7`$D+>N|bW9m3E<-64(k~df{ z&?ztlKa99k+NnHL=OD9IpXtRkr$y=}a#%Isd7YR0V?FZZX-aFgG4X|LK~G|;vc(X6 z9LzRjM$sZX&o;>DBq4t~2pQR$)H7-bPTTEfQ_%PZE|JqCr+aROdw@4N&`gX{c56vQKe7ri#4O0A#Y&fhC;VRjRCJ-d73?BW z>N~9`Iwp;R4T__Z$pgeTM0-Va?Y$s=K>RO5KB?5#T;x3Z3e%k3!X~kM*@uh^UQjM- zvOXgdy@C)?_w;}|K$m7Jp`Tb+ZU(o3uObu|GKF{{9+ig4ra1FnOGVoRbe&}Ft?Y;F zXYJAU1vaa#xXosbwq{$Fn#T)wsi0a}RD4HsLeeWGlL_S$r^e4qn2_X3i_VF69}PTE z`cf*pmTzo0CG_FTu|cXjRgFA~ob?EJkmH&0+&TUuA1<&$9sUI~h@h3rfyutvo^+?p znU>wh`8e0a?xfh6gY9Ylb7nMc_iw9Ps@^l39Hov54$~bKoBG85D z(sU*IcT}mQYJ1gXN_~K4334Z;BEa`2}$lJS!Gi@fSp4t+%pO=_J<@+XlY*-v(v|!==%3T@@HJ)OuQ?A0h*~ z6=x}n5LLDAO19Kd8ZI4_S!C8a3&^Ne>d~nl5B~Z0voC-FA_~ri6DaK0^qvF zw2fKA@Z2))G@{$%4d;zZO`#UEMYH^C$+7%qorOy9=H`>;c+(7I@G6)TWO<5Mj+u{| z`x-g+E4fAv5u1BbGOMPwN*f^OAqY|KeJ+tLSiQ zJoy)@2?~>D>IdD5tBRQN51be*BrNAQ&}L$;{9o{%U-2w-@5`;>+U?rlA>fUT5v$8x zRRYx!ONb5HM754G7AJn6273iA1^S6Tv7lU68H^62XHe%)(oxh#^i&rJN?W7MLBGsX zQd@KhzmCe?=Gtar2sMLFqgtXGv?%=s+3I501vUY$0!3Cw9o8hs(i>_3svG_EV&wTI zvJY|U;wE>Te~*r~l5wU|KsQy!0Xt$=$yfI zoYY9$RB2N?wVCqO#;a}j)wZqFwr$&t(?-{KaL)Ye{$IYNX>@1K9PGXJT5Dt09H(nA z!GDQo?1jlvvhh`T=-5Ffw??1xV9wQam@g>AO2pKZPEB9!1ytkZ1Y%pb++ zT&&hdKJLtwxhu6`a^d8Ci5C)kCe?*8+5!7BZxxlY8~iC?!7+x%Qde<1@Oe9!bSifK zcKilGmo6Kh7$Z%c(G~Wbp|RlAty&Fbo41_%qa)gW*Ivd^(G~9A3XRG~bter1=S5|u zxies5Y|}E-Vd@3B9W?L^_42k*jfU%Eu}Vk5=^-ARlG!?^yZqX2dW4RJCj25*Xaaa& z3b-low+4IT18ObT!JC^yYN4V!8eFTQ;1a!uYTrp}f%)(pXWLcbiZ~vT)i7f-tfwB9 z49g+wCYult6_7VTu@ws#7I46J&z6OHagKGd^`!NZwXm&IVd>T-3X_FQ|Q1%vIp2i%n%V6Z%g+usGa&RFzstS&n0ZPTwZ0*1|CFm|^4 zxnSUwU4z4eMuPah;2W(X9m#Jn{dh3D?&(7@ww1B}?BpwmqoC`UV_a(9V)<$9fVpwn z*4Ng{R>JlHEeTIrBCKVsb*z~dmt}@^hxMGbtgV*qk2T3^H-DCNE=$`d&vAXt+>ur^ zWpeVhq+LlTllk=XneQE5?@u+Ijpb9sIKxaskkm<>h`P{uRK7)W4tV!Yp|w=Qc*Xd| zxW&-luugi)M{@17W{TVU!@a|K7-o{A9D?hB`;2Fzyhoiuhv1H1!wI7?7?pLw)T#-N zXleR~)?=BpDl@ZNP{N0U@wotOi3i|+e+C!)D4mKvoKwM6dN-k^p|VO`v&nY_+VFtfMWnEg{wima9RuOWMQdft@t z$>GV*6Qh&$)P|V{9422O?Jw&nJVIaPia>J;N_B-z;3375W#CVB0E4BxRLwZbvPq`|R8FbD$j9gJ_`~xRIGWC(5|m{iZsWf!58oPqwiE zS8OwFwQYN?ORU{2d(558f#x#i0P`pF0rLQBrZsoKA?RwZTbmfx^KY1-=5_y@c`a>N zvXtC7DLe5+^82(KS>>ENeSfvw+#R7Lu;6&(GUW2(g&Wv?vcMP%Lyk6GC@noTmNSK! z3K`oQdl+znaHrI}=#L!j+Ge+Bf6dOyiFUZ0c|1FO`;AC9IH>l7vYs?ljayRn{QZ(T31;c*xuP11&p+OkfO+PT2N7)Z8H9( zw89=#J+XXZcG9u5?^)ZOYvqYFnhcjt7{{3p01eA+P=z;qGcpH?0W;XM2_z0Zsk<1v z8%G-o8J|gug-?2vHcfVWin^xSUuGB1NzAF?_~R_>$?$Daf6&)Du+zz{HO%g?1 z`XDwKsDp&4W~&+y8PLHRZ|KO^WKGnEp4nN2 z)2F0#Oc|1VEU8)wP3w|X*x6G4R}1C_iCYY}!2haaT4gvahKPH3J1LLTw+26p=cSWC zFkb+(3o`C7EESh=tF`jV9&ZtMg1uNyk(@3$F%F;et$V*OQ8`Fg0lQcVUPcOEg1^l_ zB#~qsc7E@lz8PLgJ^7ipf@m-LYsPj?pn!yrKD}IPm zv>i7Y?A#w1<1gSiP6Ts%AL)e(NM-a5-y^O@MLbAQ$zE{V^Kir9v$REzC8dReQmR2V zrkKB2BwGvH0b46uajUUSa!eFOinW5}k5V#GYO9jm9cQ2)wXeBXUR-l5*I-pamD@*yQ%OQ*xZ z>di#W;T>4L#h@mLK(F8);L)N30uaq;IFj@fN{N|59&raE$VH?**tQ{H^_$?_RGm{v zg5Z{h8{6Py+lpPu8qhspl&!z*jrBbu=ZO}#r6hE2Lo99av4-`IRRkup(-v);XqAm7YFR&fz~6a)9+4>`Os&Al2a-D*%|9p1OoF0ynt~s8w@=>j)-WxsE zSBq0o=^G%f5iUZ#fyx1; zxfYE+yWbJ@d?1_P2h@`I6$=)wA)+iF1Cs|)<|;l|C@oeMMeN;?{CE-yp50s8fNcdg z3mV`Q%v!6^H#byH_4L*-t+{?JOsd%t?; z%HtFs>&`=)L(_2x3hsyG64W50&_&=Tlp@!m)_jQw>J=i|me|D?^F#P2V9uHe$Mw*y zx(#m(JpODFCnSiqjY+0LmSu>C#v_()W&2`12$tb~%Sg*~)T0B;q2?g`>Kd>~*ICwD zS^-)92S4}H^hVgKZ&qK+%N!jt;#0S#yh;6m{prh}9@+cdi*0k2zO=;XuIB8;zTa2t0UW=nsC-NH>y*Vf63`=YChlQ@#>sg9!VIB#`1N*e&J zLo~SE{lT7C#*~u^(K2D+w#3s z4ytA7MAi%E>uKzb?ZCA)l6cO^7Q>%zI(pO`))&K*tT~hsk$ecBm;a7aB9_dBs%Rs1YBjMsp7RJC+N4Qi0}g!Qp?6C%jI)+b=tMuNxE-aOAd4a}AT z=I-XYmLHbS);_jl7GkJH1iD+Xc-)!3v=u2GQtzggO&&%bFp`tQW z3u0khap>I9dkeL*mvC;Y2V7@ASA#z(3=;;41;tB3dw#Eeo_1Gx`Hr`R$L4Xl>wAm& zI>{~6TiPymj&?vdzPb8V?DL1%X}Sj4MeQa z9eNWpc>n%xy4|3|%m*&$Z2ks(O^zBO%)>1wtUYZt1I`3g45$zgW;K=IfPVvaTRMm%*>$k>&r@S8MH*7v2W$0QYlO2hUt& z`0+AT%hUQ=4ef*KQ!A*W!Cj9)fB1GdxnA)Z;#0#F!)NfTw}_8OUsi~ARTAN)^xpjf z-YDVnWaYQAU3&%vM-Li9?_#EP)4pk!=yA3Vj%KZ(!o3Ncwh=iBHb^{c0fx{FS{^a{ z9(_7#z$^SwJ`)V~nNZ0P){oAlo7h=>A~`H%Nn3C_gsuyY zRx4^8jnal|8?@PKXJwPGh%eDQz&Ao3rM{%^aQd|qR*8yaHEa@W;8fbU@vJ*FXrbCG z^|^M6CW7;D1N%}@B}P8v>*>?IOO#?-IJ*s&$UG=>OY&vNMh=KH^beZ09GJjcX#7$* z!WR%G@aut(pMjFo03FUp_KCrn7RpBtIl;f-DJoip-g4|8D(CN7EOznwQ9c;dmtROgV$g;&3aL%f1@QPb_E3_Vkprd`FJ)p&z z6{^vo1vhb4ozY-Hy;X4EBQ@y=r<>jnmV+; zG)C(L&AZ9>)Vskq4$+rQolAE?r4t8rlE`J#wb+@u>jk0kIScnsHxzhZQJ?qspe z?lm_SBe@02#y`MO9+5Px$uOV_6Hp^Q1I$Uy97w&GZ^7dUh z@%F9G4(@-T^dIfp;alpx?S1HV`Kl=ElxA9GeJ2UW%#Y_gff*Zyn(=J(%(=$hhsR(G zz6Kvas^QdsOq*%T)nui+(n?tbu54@gw9SG4SPl4lq^NJT4_Zs)bzZ$5xGA+!X;q+r zsm#WMOd!k{HEz z7^RoM33dXx-UoJeL7Wz&q3d~v{O$-cLoXa@DfawilF0jn;o=MNqjXT(D%CWcFyuBG zu#c=jraIWv)?CW8(_}UMLFLtGTyC6Y_z!HgNk%u=tAoJO@gTY`ATcm04}b%)S$QFk z_SNt;le2uo<-*E1B~|`cJ}g&~C(4qt1b6*Z>ko`@G>t(V#IXs`kxhqQcp9>rEy!!O zLp4?un&Jew8(ct4*b2^nCOrtc&obCg)@z5gU>ZQLK>J>abp`i&K06Kd&;$5!F2R2N z68%5(z#U-|bbO`3P~QtadJy_k?ZW-XkPXl%J_qi<2|2R^s-p317g(&VfiK*_jMjnb zHf0N;4ljc1ie!UOWq-~(!?iV7e*+c%Bm71+C_X>I`PJVs;y(8cRmE4pc^RJeEkuZ~ zpy)maZ-@)f(=~?2su$y z!6~&5lydjs8~hRs^X))z=im(P4>f9WyoExM6^V~wP@q{cKm7eP?&G5!>PCOpgjdMU zRzaCETcjE4<{5BWor;{S9ZoX8U-n=8g%9v<0sjbp=`Pf> z_rSkdh2NY4L~0qFrmsRD>hA-x330$?=-OsLU0N3$gk~7ws!-_-g=Ts-uKy77<}bLG z6x^+j-W4h|J>?L)b;g=q0GHFb(2)7NC?7yBJ{CT%?Vur#gbS<&v?~MgMF{4nzx$Zq z?YRl+adqIA+7Hhw4t}?tkw@-=K5;*{4`coT>eLrd3O)v2@BvtE0zS{g`*HltZ1}4- zLPa1Ed+0rAho(Wl9LQ|NaWc zWdAk!`_(1mp8OvE8TeWfw3(0a-BjdjU-9>|q15#IqbtB4bbM~X^_Pa9a5SD!6z(k? zV-O0hor3TAyGQwZQly~P$U)T`f;WGUlsw3i{XI-70{;quXRp79d_O#&DY%z&=;`qa zyU0bnokjnjmH+owo{Dj4hss3@%!{sgf__)^y!cz$$S#tho=d>>e#NzA;oaX2(BIQL z8?Oz@7p|=wp1QvqcRRe-hii60ym}gDz!Rt^4?)+x4;-}PxXZJcjs6btuW{G@{%OVV zIJQ&gS^8Y(rABWe#Ex3JN!))J$=z9;Mqx<;IZmi^GxPy_HaaHgtMW7B&gof@C zG|hge@%#UOGOsYEH}U@?81cQx8#X|-yb#Z(GyFX2>J{}^U|rErLq|eq?_*9dq25B( zdWjvxn%s|7dj_j96FXrk%*2WCQa^;hat8YKLGkte_!=axRZPM_zLsGKQo@-PCwyEB;%=n##*4z(ig;&aKTTWgQ_qMU-frq z^3SHnSaUbs|3jO2w$~8Qjml*dXTyr8;BE!g7u?GB|jbh;nUI_0ze%pnwO3>4K zF?t3(ufMojy z)X%H(9iY~F0GH}d&?}+yCKT%vfJl{qE6yhTe}g`TopR@LHSp#0Orhc2BB37KX%28- zgv+>|4crf^Y8+P&Jk%Fjq<12C>MFJ3Y|X3n5hijbT2GGwrhkB3)$C-GevLL^M(!$2 z0zWB;f6D#kqk$4P5 z`H37=A)QHH{veRu7!r)iPY~anZwW689~sOSBY(*OoRNE=;vU0W`R@EkQj9MRWb*)7 zP11pD<$-&~cc|RI=rUW(jzBMd6g3qA{v0WA+(@HO^%cnVD0ZCc$PkOcd1fgbzW<>$ z;E}Km$jmG@S?kDNQx~dJ!Spb5v4=o7=Hlcy2LFT?I1jVgY3M`WVYI^Gr>h{(nZ>Qf z34Q_HkHhi2o=AoiU8 z4Ogv;v;pm>1!z;X{Lm$pXPuZ4yM1k7_&3NWVuagyN&YxyS68@7ei+6o z7xI!OLVtb>^b_r*IBB+_nV}x|Dc_AdjU!DZ%qz`L%^uSZ;}38%G7Yng!%@M=GPknr z3rq`fhx&py1XZ@RHkQ-2WtC4__;bhi6TiaKk2*5sEN+7-e_(3hachF*mLXQ0gtgUQ zigZ2HfTAO%TX4Ex1H{0q`cAE)(JA1b%yML>qn z$P)CAhoJ6mhCBWb(ZyA`QB-8tfC{JKG`NK|k_~LQ0BZph>LE_F%kZk$rj$`6R462^ z5H$epX$n-PpfHpRrz7Ei^g(VR50zU&yEsC5F1Oa=p}6trcad|~1ZOuJF+(}MF;2!q ztTtSyj5Gn&)g^Q*GW&;NTfzhRv-vE%t7ikp%MbqVZQ~)+bn_|8S!+8Ru{E{T3V3Ww20oT!`D56re8_s8 zQYaxfp-*!0?1geO{h4?wpjoIl=&(}5$TaX$_rs$?$GJI!)}&pu``R&O6n(&aY85DY&(f7zIJ_|X(^T4=3Yb+R zSX-#>CqmKkRqFy>{$lNmItErv=nP7%&}p@X79OGl?h;&j>ak{UkC5S>6A2e0FW*?G zFP4-dq^Z&|X&0DNm%(so3^rCjMDlP9;KuWF#hQls#=IupGz%=Uq2_$hSnM|MGIcjt zq$gnQjTZMvolG%S;MM`p0=tB?4`~%LBDiE=qOp!^XVS(mFF%d@Q1Qoz3efkV+!HPt=h#Rcedh5L?8r6i zvo3IDt3-bz8XG}vh#7jZ4z!ux9doJ!Elro{ne3E$ReeO4%5!DAdRnUs-#UK|3a5O{ zqixg|Am*RUcjGr=XCDbiz-8)TWvjAM&BM;<)!-9fo^Qra1v_{goWvr5%w-7+!Ffe5{@N%lB0{YN`_4F4wH^G+QjRhV!K@V6Hr1)xy$qKM6OP9(kT5 zcKEdM^XeZP(r>x$)7DszCB*vBzz}0+Yf7LOOvsVex`vT#y{C+QZDxs#2^q&7Z{5*e z1AMTXsq1M?-fD#Ru5Fd|m*tYBjj54TL1;;;vl-elCC9tm{l|ISQPo+&!}~H7A0xQ> zRzg>4mNZ%_YbYlf#Yuc#?jNRW>1toN`EF1bVBDXAakCtnBRks%eCZG$2~SurcC$q+Ir)>zdW@)Wqgz1lboiU*A`$jZ;|?ZxDd^6*ORmab*hr+tL3{RFOc`i zOO$NQglc;st+Vkn8~>JHMMouvD;i)wRlpz^cTdVexg2$eP&j>CB| z%Tn1o+S=Wc0RMI!3^arBjbVy`81@5Y%x#?$a5dP7P>ljWJN6Cf5P6b31Z@^?<502i+tddFI2jy^{Br zd_hZN-QZGHjQqnN0rz~4G}chnI2g6hXJoPdLeuDMHkj_v8fbk~Rh}-};OB5%HDT3x z^(owPG7|B~KfwCSfE`*74rd+6Kge_z0Hd1%=dmMTN}NIrzX8!}D)9Ga`hK;RT9|s& z8QyT;ZFz>g*Y{mDKq-G(aRU`^irR7;A%;K3uCkswr>$0>(PQjCZMHTKUVHttA?z%o zw~5eN4MW{?neYTm6Y|j=>UsH_FF^jHc2%l*E<%^K z!ui8d-_0qz~Z#h`nB`(wUz1$Z5y6dAvz2*>>zYv zm6#u=Zi(E+gpSLbkv}zrE6)Kqh@_!jyA>+(U2wYGfUYI!K)3DiaV*E@a1X%s7!LQ8 zZ*XgB0S@gzz6xSvjs&9e)rkzmPO5Sj_ydH8GtyJwN?GW{6#*n`4@u>^@zco+RDJyM z`3CSn764bZA`Z_WFK`cI;MlkeZdezQBQ`|GlsfP*T8q=S512^R;BA$M4B$3{Z*hfv z0q-Ig*aV%K4<|%R+L;zarC<}?Pao0TtRR~Lr?+3o-COEr{RQytyy(0Wj0jzZp5--r z4J_{+Fue?&VbB=}m|u;41ZYe=Ftu;!wsRU^+XPO^ zVqh_T4)=I)&gLP~@%K;eiMmx3vb6%p-#K_#d_g9>9^Of#ahEOeRttBZ2VPlCG1`5B zcrM0x^#``s6yK{1yet+!QBTj0aq#1zgOLaI1s-@5D2^9dU=*_TC}1Q@@QNexEaHHj z^g*sX7KlQB%rHM9)D>vf8F=UZ0+O%d`rZLM)wyJ>5+BgP=kUc@3?Gq6xXx%~{Wb6w z3S7&Ak4|LEx$ytec!g@fZT!74+W{SDhn&3ta0~fq~xpwC$bR5%lr3>#3EEcmlc1FoQ83~L}y@ZuhN18rIi?4cuk?EKt{A-F&IA?pXZ zUHTc!<F zI$r-W{(4uasY>AfHV{ANX5dG_SMOiEMh;@KV!#!KaoaI|_YlqQfb!y`)`|bamf}@i z*pbqSi&g^aHIoqd1Ac?rxiUY3hLDO(;LBh~$)Ov$Htee2kXeCk+~VReTN(iuibhmj ziv0&(aCP*b@jGbDL{{LIICwek}7p~a*!CYMdWT*`t&aXi2vJCe~ugTY-o46`8 z8^7Vvu92fOO}MRvvRB#>t^{3-URzawM$H2@bc|*RHEF6o0dZKeP*_3DADIj4v|5tz zjuwRT_eb8LwU#z0?Tt3n=qKyRAVcrh_2m%v`=HbF+ze^aZCjIh44nA;$4G#V*73DHXUtzP7*5+I#t z|FSW{Px?-{%xY+Vz`Z$(%1B$bLM=*%YhS5ZndJn#KG3Utg@*v zS^daApppCzt_aavHdJsAfFRF!R`;ai<#Lc62rZ~0KfS+RT#te^RQY!q%Zv$<@R zh~4=UTLDjEjg}%GSzhuNNcVQEsysl~h9e)0N4;quc4728K*B2vxDA=Dh6^aJxO8Wk)Ku~Q5YxjCdDvA{MmHgvcQ~T`e-Q%om_n2yWk3ea!{*~iJ||73=bI-Q9idJUwrPZ0hK(t z9IosP$AFAZ4oh~+KYh|e|8`By_j`QG*DR0X7TXA+q@$wBQLKXDy5WY{#1v*;3tshc zzKC*8p2X~W9z<6^)dlWN$Pg}jE6H(kqHO84!wKrI%}MK1mZr_m8k0OQH8~+y%7YYRP6gV>vMRht(5Jvc z*4ko>a9T85mPy5os(6okr0sI-%#{D$wNG;-cuLCa+@~EzPd}wTD@NuDoelq>(rlM* zfmQ4Vk8&Xy&wbM}1L|29Sn39>K(6~B^kB$F^h@d3xeM5FB$LYV{vNmNUrMGh&Ggm0EC9-LumbrSFo9GSY{@{tXg)cj&dDR`x2I<3+ zSi+n?vt2#3DAFAjbw&7Qykw3rtu*Ab#st>3T{2Y{qgiETty_17cz1eBd(&L|e5E~$ zoOb7KcVkacd9Rj$)4BxrM|-3dMwatUd4@P^H#w;HroEMuz7r~e=ju3pI-NuZYj@}& zHA(HNj?#LtHi($B5%ElCbM@yqV>bX3oD9uLwDC05N^PY}nDrBlRgKIv(CkM4r-DJo zu%BV8a-GZDDPP3`S%pRvZxlVR!n`t_^Dme4ei?tK@x9wmp1#YM&@Z!UPP?q!&YzAp z*`Jjn!IomjqJNYPk6aVF*04f*;C}85&Uux&6%GQe9j7yzCO1mVojlIILT({>0=(f9 zBZ4A+hF1wlH~I{2Q;g}H)RNTF7I|5A)$B@HC$eMgO`Q3_!5AnP&}5~8_L}wKM@tz% z6dN&{dQ;Brqj-fIt{+~9+CZDAn!&NmM~9>CaGG~OfBlBP&A~BNDJs8J@}R4rNBc{^ zvkb(Sr-9C2gkw#Paj&(GWuI}c;k#j>;f$0=(xmdnH706~u|Bc&4{QXtfx)@o6f{TW zD$$|zigMQ~o-f-mvY$}yQ|@cuFVY+4XVIT~{u!LP*b(6LxDVhQTob~Iw5!ml%&21d z3X~6+r#;C@&#al=EIlq$&zzRk@z3w1poHP6iCKfaM7$h4Az$YLPYc$`mlifJ@TzIA zVY)PuR90_zUO0|qa%qt%w)EE-VfNQvvmVP&5snEh#b1Jz^n<$WiJ>*$hn{mk$sF;w z^xp;c&n~ZbtX7ZZAx+?@d20?|9m~_X42zb(n9o^W{4-@ zcwr`|l>DxFSv4|-WUa{=;{8|M4!5x@=nc0-iPg$*d&C%X8C!nSa$tYi=xmXnj1bGB zUKOUE@pN?-&z_bwJiDiJth1auUS8#`=zQQB>shHRSIYxc>?2%97WW)jM7n;7e~L4y zlHP!|)v|om<;E-ud*yv*WOvb5Vmeq_+bMcz1C43G4}ovPXc8u#!yNmBDnPJ-`#=76 z#299AfmVFUI>Fk;_R`uVpmE6UkQJd1;o5XCq;93TYgn(w=Eiarn)h-(b(&1*zEd|Z1Y!zwL4 zk|KpM{9C$Ce&d@jZ-Ju4Oh2$z;F3S|u9ln2ljs2bhqhQRt@f19t0mAOL-u8%r^p%K zH1N8Q$d7&Pyh-vc-!)oIZK2gb?3Bu?;|!$Q83r_ukHtNf)5kz5mBvjZU*Qt8njC}* ze-zglsFREnq73k#=|E1B;dIsr5$-Vl77 zs5_6>?&Tf*?LD1a-IBYiJV$9L=V-HOCm?5!QK#_f9{6ak;1jqlWGA^KEQ5~>6B~=8 zcpLPFol)gD0)9WV((0b9lu17qwe*a7d*ORosfv>2L?_EVn$ zmE3}^xh>@$%2zp3=?EX@JxV0pmJTahw62JZ16XtFW$V~?Xj9LCA+6)7xxsf?ihdsx zf%>e~pV1raEW9pXYsfQbSvpQ@4E6XPtr_F#Aa(;0lbiKpq3EMof_tb}<6YcVFsQDQ z&Rhw;BRViRNlWPOV^Q~i3#M0ppbLkPH!X%YzC-UoG)@FB@|u2!yFlLQ@n8>u9STe& z4=D`92s$EtDLQu?2A0^HZ@}&5N5c^&i$BiqLM*-u5&3LX{N3=*yaHd7+nj^s0$Z>F zIYO4hqklY^M5e(zrX941o%vZ%qX&ZVIG($$o46aSnO+wo)stO9XOS54^o60yDl2epdnP= zUk02le+QH0|7Y*yNB8-$(3#eRe@rdZsQZCOX@DQO08}Y22J{+)e8{^Y^O z4Djo(v2^tADTeHMh&~=Z*OlRe-w}G%HhNn&NB>9zFs~Y6J*-0Qwt+qjIHH738?(TV zSOA=GDzvKCp}ZRpT-*SBnvz$T0b5bWDvip-OQ2dixDHUeCW77L(?c+8`T$jIfvz9J z;fN!H@2h~db`=O+DA2aw@QR-YUd|lk_0TzzY`qmw$St~)Tc{U>hcgGnTLI5B8bA3J z44>`Loen@HzAdoak-%ub>&4*bn}*Xp8+^VGSi|4-Y9v39IWI^;kCD%je}CCBM@B)_~#5DpM}5;D9n~{ZNNgE z3oIj@H3l-WNH0Vb))Z`nC*aymWjD!K%c|7->h|IECLA+8H_m!{BM!|u005?sK zBW`%d4W;9G87RsV?hxxlBt$a1NL6|i9Fa_PjyZ$ADi$D9MTN6kEa|}-p!fJ${UcE3 ziKzSsl8SU0H-}c>slJHyM9rWsoT+=TDe(9F#@%L{#OClx9m>~72O}pcR}WbYa*3P8 z%zP~_h@K~Bpbej-52J&H7qk|T!dYAt=|GzxL;mCw#EbeidKUgw_4H9#clEW|(EChh zPx-3!GCG+(0xnz=bMhUOT6T;~0NV<7>;x8SFe=x1@Ncmi52L0sL|rNr(OUB-NW5BF zzbZ7LU-cS>4{8EE09TijY6)~SD33^`0{>T&;9_$gF3cf9X}y}(3VbaWG{a?BdzL_M z)5U^aGZ0B#CY9$J=qlaKM?(kP5!H5L=osEY_NrT{U1~3mkuL$6{vw@F=h2>gfOr?)kALAf5kMP~g8CgG+7dm5 z^RNN&(o8)S{irnBLTC$w@}y$d7xA&;3~eZxPfH0UKp@R8%%>OmNAha8R>!gKyi5Sb`n=6I+Rzl16vqz9O+E4wHK1GWbc0F;@do9m>xKW3P;W z&MY5E(8porT@?PZTw*)6NGOKBIVG@0;`mEkBygi%`fwo8Uxe{wDW}qCu`BxkrCKOA z5PL1dPEj4|*sH84_ky~)chF&PR?Yfr&B(q%GvrgUnO7CGWps{mk4;ruvuauwm4jzP zRrqB7OK(yGd?vPnu`eJ7{)`B87`Vr4aP_-^sc+-sP#wCUKjS%p(b?plHcChpdO`*C zmwgfwxI279ew6f_FT=Y`_oQV0zSPZB)>Of=-f+({(3}nSSAX~-R2QZg{}_APmcT!H zsJ=i@!1mmZ=N_t!rj^tX-#?x}uhSFZP4M+|`?8NggZ9Dm(K*?b?w;Ttom@ zfY`tp@IuTVvJU;}kA+kSuO2cd;Db2He9m|<@T=)Ce_CBjqgiw9Fdc(Reu`(CXNt3? zeTrkay)5#i_s*x8@qb18JJ(RpSf|Mw=y7}2ddng!)RkO7orj|^tAUkk42&BF9?XAZhPu1}lOcHk__eeyUHsKjisN(evC9 z>0RyVBe%=`FXKk0!9C2k&bh{;I*Yk(x{G^PC~cJ}%?CH*RrC&ce5X*?8YFJSnJ}Cc zB`x_wP!#RL8C8UhAjMgC?S^-vR#NUni~ABiKF4(LOQoyw)6>E?%Uc(YhR5k=`K202 zoj}aDYlE1DN>~$P`7WsO-6vrMN?@7!r|nD+2S55||qHC%8f0CE@vUCrVQTvrQ2Zsn&hgtx$skPmt4c)mIs zdoH^V`wCIA&rAm&|Y+6 zv$K)t0?`nxDb4`lVOa1DKS)(O?bm;Rhb+%3Xc4NhXc3|WR*lEHY?oMXvv zJ#V{b8x(jV@Lgz`u>Zo&hiwjD6DV8jTMGxi4W4c8Wa=!%^BKYgeW&o77NL8zM7ggw z*8R{~#P!=1=w9fYoLxEln*D|2nzJPo#qS;Qj=s*bUc2Xrx2W;}9tK^M$+U-77+hX2 zc(FIY$x2|4*&$S+7I4w}e~3=M(>w6g2-R*X?dUz$Ph-A!a$7A@P1hc&f!ZTwj%Lvh z)2rxT@e+B`c6h-zXaB&RWjm_U>rlVEOCH0W<`!7U2f)AkLl*PV;0_Ata9~D`zE|ud z>@#E-zL?K|w_D2m(e~Oh3m!hBgKEOJwPA3J(6fP;tTJqaCfS-nNnJs*i95w!!Xo6{ zpY(ieIW3_6P^S3~%9nk)eY>DU8)PqG@9lc$EahD0d=HgyKIdp>VQ({cxbLesR9Wrw z`0UDUb(>Zem5noSV{d{A_64qnK7!1IE~hb~?k#Yb*$m3OgKgo4u%-G58l#tm6VD4e znZ|MD*a%$!&(R4Uq>J0mz9BE%!&S!!Hh^NWG91JvKs#HBrP7aJOsv2&F9?tC2;>y) zIYw#=!?=g!tRRZ3ghNtKLo?%V!&_*DZ{q%At=ZNE0eP(9fjfe70>gq`!3{&IK#|_Y zGQt*X{AH?VS}k4?b`e#`k9icT$FiI1J9V_&6uU(^Z>BrhRm#=cdBSvN9$UpCaM~)fFlngDfB1W1Z3qs@_2Onp*{84S~h)0pVF7AS8GNi=_p{xchufAh_=%H zMJ~Oa&PSJ^0^kA+MI~|;xCl1no_WRYz-msR#={tsx)J?dnwoY=`NU#|gTf0^h`YdM!jX)^W5SO8xt(X1 zE6%ysncyhrs^E0G20D4yS7!xxx_gYfmFKtTq4%Y?wp>^~C4Z2u=)*Bxt)r%BC$vC% zU2Cg7)7ELRbTm%ZZHR0CW%p6{i>604o({wr+6H;VS?p@Hkv&-v7cOL9^loe~`tUtv zOHmbW%<^N6Z)CHerfZITHW~Fq5&RA#ISS3&VsL}tPs*oUtaQ;yVD>?OA5E0UAI z-zv~sYNpyxp6+evp6(jz40b+stg$b0lyqEj6vAhF9j~y;3%d5Z5?wj&SDr4uT)seG zVfl-ETdAkU!<{V^9e0kQ3)v7nK^NVP%)A>iuCq`OCL+t94{j1cKehAhBYYh!Ks1_R zzJG=b%3)NvXCuRZj1%Ak4Z<8>M7x9IbsHW;J-}kP!N%b3CFHNo5MMxH&TWL>z;krH z&V_nHkT_H-XV?lK`^&}$rpM?je#{bRU0`FjqXEHzvFHwe9lb0DgtQ7B7+O7~T+jy`20WzJin7>Nh$S`jD7G0LvKlHk7*KG>JoW9_%?F8e}9 ze@8FpWT)hgaOd_k_O$f|%8jrm=K<1I{G=kr^7HZ57`1X0IJAoIEk)f&w4{v@&;5Qx`{{8%V(+K5Y(Yl(aoctgfQb=`FCb1if=b=%#4+_}A#y}RHFI#B*9 zhoisPF|`*hPpxbuuC_0BurEwQ-^^I>e!fG&WCU8^0z;@85Q0(qCOG;PgNpPEI(Q`0 zHkc=|m@OsQFx-0!;72Yx0v%oNqh@@H9-|*=Tl9E62UIBonzSf%R#=SOdnA|ymB3`O zkO=sf*~uI}x6o1e2yMp>QHLYzC-~oc4X5BxxY=YfSBJ8_IXWHXvMjXB#|SOO+dIo! z%R`F<*O%MY|E!a&R?A6KobeO5kA(_7K& z_ITa9-TB-v-QCQb|GmiTM?y1L183%O;Cqk20S$#S zbbsNeFbODdXG9BTX*jw9^_Nb;hxG}(NxMi-r7zM)sgtxwij~U1c_m9MB6UMY>a9SU zf1~Sony_6c1IBnsp{p>7hk68fOMRS4e}Uv|1wOSNypbmC7tPB~p}*{FCo^C0>K} zytRIjt$RwCuM?O_USRzTpl;8>RqX_Nx*cCTj=9zWjM<}LL0$uUW&%1Y^gxVP8|a!5alk2@ z)uDK6fI0mX-8Y?>^8?Yls|t|Za%>Vbrbn=w&4#BzFk-z8Pzm;i^GOJ52+$8g zKH>kbz;CPot`3@W0AGI%e(V5OB=m z@E};nuAyV_E#SB-@mmvc@}Faaf&BW{v=8~sR?N6n_)dBK5!;RkxjA&x2QWJcUiCV+ z9hKv@;1ce@Yp3b;u$C_XQEd*M@(!$mw%|6}p)Wtm`Oy1jFE?0!%FRH}N(Cy*n?R~> zf(e-gj`BTtMZV`FrLZC@31~j4pMpuKLLDPj3%2AB-}v zH2;FN*#PH$K0L!{{F{mD;YRdoj)odo=C~Fw008FI5`Kscm|Y6W57YtNid%HP>fi8_Lje=A15hn z26?0x5(<;t;14e3*73c8dDrK?+;1`f+{rQg3f3Mguc)G98vjJ^Oj_wGM(P58jr7oq(~jVCA0nmE<+MAClcr$Q9#MmIkT0ku zk#ndiZRCT+9&|KjHyw+L&qd3HAQWJ8_mqugAbJGCYlkQhWqL#lIj${3~i&&9%wKKsaW0uw;g?vFp;o*g zTcx-BZPr4IGdxsgvKa2BLF4k#P<1=>1aU%n(#glG31qls85gC#_mtr08Jdb?Rgc`3 zRyO_yf}E=URH3mk_G4!hGaJlPu00QoPyfcZV2|J}SDf#prwL!=3jb^AEWoU)zQ2F+ z#stH_(A}VPw{!`DAjnTbDFF#VLP|ioL6mL;L1BQQJEXf)x`!DiZ=QTVXa4VVpL^%d zJ+=4RYp=cbitiFGc81U)^tIK;wcAt2=pO87{U|nbl~O}yRo_k57IU~ah|T?C zI9ZSN?(!Ve=LUD`GPN}BipN7mO}Fs5{Mbs7kA~)ez7``d!^7>dvKy22n(7a(ZDQfz zx?nG{1NgZc&7XoBw42fn^$YJE<4ndX?UwR=#7g(9^gjLxb}%|k74#Y5Bf3dm#$qu6#s}zl#Z+-gY+r(d^ukEUSBJ=38l(WR9t;>FeITi zx{+keVeGu_i^`4Qi!g{Dku~HF`d+Q1I6yw6EVcS-pLsjm&Ba>!Eh;C^QJ0uiv>xFW z?k>s~>IC1waJ&`FK0>afm(Sd%_UxfwJvi7u!%A@PbL9~C1%pOgy`|ej4*Fw# zlhrmfRSCFPNEP*i{vDFynyl29mTR4i3#Q*S-QCPJB(q>RQQR6;NENk|z=H6^2)}$$ zX>DavduoX~-*r@|=YA3jTVIAYMBH`#X8)-5FzN{1Tr)hK!zILP!K+lKO2M*P08573 z$RRIu{cF!N$dgmsm{XN2#%!q#QRnHz;ZG3{>HQ=y2;|B5g*#YPtR`xL-AhT=3K>Pr z{jP<|zd|_F)-I+si)gJRDV5SsfjZmPlgm3!=;eD6o@Q_H{Lfuh>=(EjN)q=(d`?cu zO8-;zm3*!RVDFql+c_y#RHsX~H9wW%R^b0%D@`zuSta$J)Hs=<##*!WRiPou9V#$w z(CeB7Sx<5+W$j2wv(DP>_0cMpVfPTt%}#Z;x=%VRJPR)Z2dTait#^~^n1|%n_&I(w zDvPDnc}Q}JeBpNDV0oxo)cB4}&#Sn92IMHsy5#^TV(+69plpI2a@SpXUP*1NWZ&$iY74$#s+2%Ob4Uat7 z^6E|PN^)-U_@C(QMcW)9N7J{R!g{@!e2pE#7vVv6Uol3VB3B4&!Z+Go_bQ>Rnil>- zIAxTO|1>PwrW43ptgiZ>Fjr~$kcDXuO z+w_&%efgDC#MN9(4Bxk^Tm9vN!bY*WwG};b3NZ^)?1#c2@s6-ozsqjAk5C`a#$)gU zDiaeCuip_vR!()D{@faEnjP>ZDzwiuFB*5Oc4(gaxKC*;S0|@xsL{sJ9@Rg^es%%lmeEldrJMlsc$=1FuQbajoux-Yia8nY%P#S> zutdD4e+kA?1AC2pmleB+-VDp$aAgs32ft`F(2F-po#o5+E3*3jvr@%Z<{qi*gKq}tv?c@-_mSm z|4qh>BF{31NI7)sybHI*LQ+M%)mN?3f=k&?-xrz*WIB8$A7D2UO-^H5p_?^V8BNAV zK|6*$PjhNURRwP{!+MP-IheXIjm0+h2Y6x>R^CC>+qy=pvx}ff|X_jzG8VbtG zzjgyDzc^7?Yjz=+wF$%>G$7-*FKwTI-t!2*oDVzQF5;OB6EQ;m zwkeWjorlcAH$n~TIXjl_pv{0kNVeWaA-gru>R|jO))k)1`^epLTk*zw{K~cQ)J+zA zrozqvd<$VCQT*LOCO9GFvT6&v$askb$43%f*!V_*>#`Vs`9O9K#l=LT6-L@*9a;D6 z0_gH zIb+%Rb{A^cx-g8K!A$crY{hkm$xzAf(rEW=_Hs#R%@y(Fr;^LClkCZ*#HsvacSghf z18u%LN7J!kb|9NC)hcc;W?y&_O}Q5qjY?Km_I=4bDKBy1^N4H-Si87-J?q4;)>`(u z0#^S$)?MpUYnA;i$nH~+26c!;m}DEwpprxsEahH1h+-ITSH=e~S`PlhRIwsh5BaDX zI~ly><>D**OREQ&_C3I`x^DSER$5K8KreQL`9R%_l$Hq*QUDB$V`4$#fG!HQw19Zj z-bC>fk!s7E#WVOAHp*Mb(QG5H0vE6;d7(ab`-1X^oJ1CFCo(=;f$~sK9YZd~OkpC( zoj*$-#D}7nT)CxIYyAIzGv8xC;``L8w1u6xnb8uxp}(%+cf6~u*XQa7wJCZ<{6DGs zuT)x^h4pI;mK_V^XS@dFqnxIuPJsTzvuA5b*849~cBvHAY*wJz&%qKA12_J}UUU+k zhj#duedcsLurr8QyKg8|jm(7p9 zwq1?8j%3gGn0@CR(3J1<%$uO1kHCU*m=V2CRp;-pF+~uQGQu*6&ibAmWGVQ2kX3=* z`+1_HqFINAV#i1mo)Ba9O6muF`++V%4K{Fe1*NbYCudTZ^-KAf+){~D*D6<)R#=%T zE9KPTYISwHvRwXKUQ9KQMdV0lmy(E8YlHp#ICk`Tct{?AQGdjEOOEhl+iHp0EImbEi9P8FXp8Yg+ug7RU{_vjuLVcDmgobEJwghSJya9B*e$7u zEXk*(Zuk{0lI!~vYf@h_aNCf}bCYL~29W};=}v`S&8J7En6*}XW{%7RvZ zEFAU8c>S4ZtR%8u=2NR>6-Z9!h{h^N2H_~;cz(ej{}%tnAS@(dVxf8x(=-j6c_V1% z13C4M={4Jfrty#{%TKJ^W_@g+N#<7U=Fu3~ZJ!h{uW{@8|&-6muE*Db&E& z2xes$5D{BQr-;zoLFMQnVE6nksnR3*kxAZqDKhmHYTzA`su8<;S;{5X!A>eDca(x^ zcW?+()v~T9YJ{sh_~wmWqm(JEcmtJ0rJYh*T1wX48!N9}2CLB)YR+!aweW#(FHmm3 z)^lrnLq|g1@VxM%@BrW~$!G8?yHQCf@~3e(#e*0+pwORz8Fss5Y_4uB1wNJ*W*|8hRt{pR@0DK z54&>#Yq7Zwgp%H7ODt$r&E@7q>}Lu#=XmUn3E0`j5H-=9yxoDs7T#nn+b+ng(+9BZ z-^ErIA!+1p7b2JbI@U&=QP_#S_zF5#YsR4l5}+7W2mRt1;&P&ith_1Jm;b;bxL$rp z1)yl^u0D5FcHf~Ud!oB7RhYj6N9O?kiEHv3>2GDJ`ar2EU1!f)(C$mj639c3f6JT8FPnJQayZe^t$O(y>DVh`+?$FWeH#o}obTltGMl&mh@9Dz+g8O)$R zu)`lE8vLl49~phuN+eS80PA3Lq|P1iU@oyzHKH!j57O7cOFMsuQ< zub^|XZw8Gy&fb7sd$@VV=t>=%*~Sgykue)Qn2uEGSWI=Ocg^#djB@jFaDECvG5bk_#!f%sq5_{iBe~L%* zTPnr$GJY`Mn({VvZ`gr5i7l|QFBN~M2mR!!@-lgaoKM~)wZbp3OnD_&q|WkE z=^awN3OmGqs0dPC>I+BQl00z!Xn6|QGjo+Wcpq*kYt<*N)$Sia{n+Zd=UU+U)blqL zDn=>uu!n!GG*;VDaVmqju(^1PYomQMGRy0}P?2DxVC_&u_~%eYAkkkiaDvK~(}M~~ z?)}5l!tJ#rtq=J{1IfQAZTG-udY={-VRn`wxA0di@bPd!l>D{)Le8&TWhBahP#!}? zmEK}mq5$#`b?_K#+5mVi0ThK5`1gK+>I=-{rXL^lX{5mfVpq$uPToN09nUN;1itq< zv6eIs-|Rmi?9`^(c01`u>09t3{(%0cv6D8pmYOGx(MZtqmTXSeXRy|PLUqX;<}MI` zw`mnYqPn2JG_DzE^@Vz8V<~u2Rmj2pVC}@7yH_Yp`!)(i#Y8b6?UAPwsh%D5;wsbt z%cTsED@yl4h`0cMwPm%8m1;8El7z!}7fNHhss(aH1L&{wJ}3PxH%8_!QQNsjxe|E$ zTz(L!$GAVb$GUa32Us(2uJf9{wqm99kPH8~QCc zC73gmKa>{S6>Jfj67H>PctQHgP2~pi zFu9>zT>e=)gU`W3gn1Qqw;o+ zOZ~}p1n>P!cM&QdlyRMOEp#t)pLA`Pk1%rksWy5{N+8-f)!b@SG`z+)#t37wRx=z5 zs-a-;^N<;w92_6~G$fIavNkj#6dz94Zt5<0{wAK4CfJV$3O%Lo<-ycXX(&yW`pOlR z0?K6&?oP^|v%jkk=1MPYklR5}IE4MXuwC5xiI|jY<~>kRY@;*yC<_d~5hiN?zR4bw zJcs5Dn-Ta$TQk>sTQks)QtaYrmz~j{UJIW= z+2d$yC8Xx$J6~j7nIUF~Wu>lC95qnO60h0>{~!5d;4u{?=COSiO4%LpYxJKA@>j^5 zM5>kE$0K}6?FP5ca_@CFa6bgWuCjZ*`=#rh{7P&gO;BWYJQ1*!@l&0~Z<%0RG-RW^ zriBKEhJ?n2Lcz1a^MT)jKa&4*E))^U8|tWC)rYfIbvK6)@$#Jg`A9Jl|7wOb2b`b5 zAT<60zkelDNmqF)KWDz0Xl8lE!T2xZ@htVRhM7~4D!cIDRWLKik|++^+D&qaM}u({ zg_cA{Y*pcTb_wTyX3e+5of5CrS9p;Mv#!Qnw8 z*ff+Fx)jKb|%w1X+(9(oLjg^2eF}jrmbaivR$w=N4@{do{Pv&S6zX0;r$29H#}acAy+4}uAs$_^u{p~~Fl^%& z@WFeq%lZ?I{kAy;y|)E14wczE+2|_x+2G7a;!9x;7W#{5zlAn-$y_pyL zsayO?{Ezk*As=A^Hr6Y|q?83mbC1zTe;&RWY7+iLTc@?uN3$y$L-a~=_#ibpO)3#I zHyfhc+`{hM3k;VfLKQGTiPUC28-r9+z$bcy6en8@pW7_xx)4w2Yp`5y;ESJ0#={MK z(sQkNtCoG#>I>3#6(YaF(DO3h&C(9lpyP8^u$=ha8nTx<4(e)Cq;d~-{>$*CmSeX6 zfG_$6HKgiVZ%vIUnImR-_a7N}H0?&;^~T01?oe z*dtb=Hg`Uy7f4bKlO4 zs%(*ps#O%9(q7#p50{^S+jI*Gw3K#`6Lk}WhbSyV8_4(Affb?%Eq#dmTEgpu{uO&q z8`*qYxAdZBA#)yxRjt^)?KX#!k5fA(OWjZ#x>L1}n1Dr8_uFe9BAcWQ z@eHl7;l$yG9citW;@G8bA(vwlzR_a%2>a=^#UXgK=ZGD{OVv$APUE;TDDg&ewcA#4)ar zu*9C_dPh!B3o)BIN?flk@eGl=STmIf@e{QG@n7J1im~=`p($B-@zy1wp4=N*Gnjri zq_)sM%!{h_L@eXegwM?W`g(Gg#_QCgGj>=}REV8r?KEDP4?&0PZ$x6HIEy}(Wc(!c zHO5(!#Gkc*5HHR!Kbm{w55f)upBJ9|*7gCRg0$8yZ7r3HSR1UBtQ1&s&7-p4*oF;m z0W&eqcrGpz-WuhsOY%T#lko{Vh&yDanbKOLqTN%fY`hR=(z=SQA=Ru;7%$OE6f1@Q zh-|6P4=C;>>Tt6xmbt;&3il~}qcGi*jDJEhzH{F-ubL;K#Ee_+8~PNnS~fd1-m_Ku zQlH>$5XkGDC-$)}xMqiPx(|yx%|7m-!Ykt#7>9SwYw9TZn%-EMFE>|anw70H?w(d7 z^|@V4kM{f{LshV#0GV&R#;mSScS*x|MPhSY->zbp4N=j}0RXFTT3&!i; zMuq$h^+C*dReHSmY)oZ``UDyF}Uy3(tJiz?-`g~DCe?(`1oH<4FE zWrQ-}eeU<7>F?snYjc`hN-rk5SgC$4 zeI~Co3fd>t7G@)=Qw_6j8^55}AHmx^T~fsob|o-{YFRO2J!yupO&bf?mxdOKJ= zO2pYKDx^ulN6K(@kN=8(K)tW;1eGbyKhs+8-DNEFm$2hKt?^cux1Z@_g+gX6G>&FQ z6YGtBNyrP5vFNTM4h&8K*Qb|K+Rm)n(=boc2 zlB%djgE^#z=2^Rk=ik5td5Zh2Z;q0jb)v9RKO&zK zuZKDapF3ESS~K&syi(t4)KTYK&q7PZTxvdZoL*0QkA^WOP)qvH^;0-byXl@MzYCtz z52<Hfm#WULZ~> zA9*hPuiiy2qHdM9hc8ME<-At3bQbIQLH8}9e~T*b+*QT$#OuN&#)9(!e@qbW}p2+Hx^(%dk5XP$SeihJ(G+Ojv_t_{+MfJPe;# z#u(G2%?{$e(aFf=j70O*gjf{57;yRqKWnjDNg&$b(z@I zL*_*FvRO;mBy9^v2%DAO?0H%k_r$po(=%%c@t(^@PiepImb0tt4KH=QIvLgFM|v-C zX`7pW34`^ga!%Ki&G=$gI z5cx05A1>vIM~kUt9hQf>N>j6GqME~u5<5wU&1G^60~?*9=?lPBduFb7*A3?p>rw4| zH5I!3%2~UXu|gOtwXnKcrNq3*hR)Im<_b8K%6&c6H1G@^5|g!kLM1g2Ocl>sea-UH z7%NUMU~g7j)V>OA%#TcsG8?=0Zccy*3Q&twZ8_a~_Bv zDfWI=;yR)}DWmKqBEG8hv#?6KDa^r-R9-mBF7t`5G7=li zdRTC~>XnV>?Cj(9O4eO35$Syk%QG5x=*V5#5 zN0?)bW#SHSf^Bg%b+ZN%(Nh+yK^Gy)I&7RaBSoL_8vpY?Y-wJ!{HtiC#iYwbLzc1s zlrMl=++3a^w3ok?-caB0j=T>)VMui;Ys6c2K2K4(HMkAY;4bGS>SMI>(LGD;r&dts zdOIjZ$?)2sjB@Q#YT;YzhbL{dT?!x2YYWSSHPv_&UaZg4mWKyxGlEmGK9^_G@J3g{UlqWhO+xD2|QS05^ zUG2f7{Z)vzz7(*?U=Lno4L1&gPCO%U$+z4W>;En@Pv-Bwy1}D?hQU#RO~JTe88X^R zYK!%9%*@)t0C}CVMy;d#Ozy{SHK^8iwZMLO60SLl56bI*3I za;ctPRO+ngI_;9N9Q@7RtFLQ?>o3<3PZG5RKlQfpu8OLctyt_&IiKd2PI+zi< zV;EKwu`pHnQX_UoW{d1ab-&l%#op@PwH}{)lIs_BvHXnu?(eDQl@JK{4`(jSXrERw z-TwH)$An}n`LkqCik(se{Gmku!SD~^Y34SxppL>)dA0IZ4Y_@uY2Nytww`wGah_VP zuI`>7_Mdd+bgy(}uxH;z{mdqcORc8lQ-`YE-H*LrM!bp2ovm$bi(DD`wiOH(i7DDI zXO*a5)Y9%PQe(47uyeSp_Pbe5iw{)twM^AM+nFbCIx5{se^f9Fa4kzlnVpGdCtZDlXgS(MO_AM`ZVWP+LoGyRyopl{GIXGe~cKO8J2E;E|p_Fs_BZO+Q4q_uMs_{ zG}g@%?OLg{RmZEz%2D#+>L^RFwjaO_S3zuw{pf4#U?Y__t|y*-Q3qpg#nz2GmG@17 z6GhGyA5!u|VSkR)h@dOvJ}l&>9(P1wlD;7v^yN$&kh1&zo;O2ZuS@LqVg37siOo`i z#AY4Q0)aW&bE^y0G|#(dMm3M=6K`O!O)gW}l zH#Vbd%9`XpN!ye1Ca?X_GjZR$P9F{?y-)7)@p{@mU-e*JeYUkgDyhtL{pbFLY=uSM zgWiqaQV}D)%iM!q1*r=6jdDQ#RQjA~_8WFHYIZ-zUY1T~NmF*?k=WeFknxx1eHm3c zW@GFZajrb`^0^A#FZ!y~^pZKsE-szDaGlt6_Y!wL;jd6i%Dd!FDS!JCKkQCkmiR7X z=A(gc{`dZWZ|c8KczZSZ$JD0UYkjx2*SI4U0*z~!r%1${Y~{0mll^9#FZZTAE91Im zZe4KMIAVI$#{l^fsU}+_A{F5z`|^dm=ot>k83psyJV)O{L7m=4G^i%|;<=Ypw|2 z($*3ANG^{gB);QG2)eeYM1rAO8Fp?JG~*R~e%Iy9kZ3*H^%g zUQ=;-3PwC9kE%w@!fa>Am`irYxyWs{B`Y_cuqRY=+^%g1O$jawk)Q)0XQEjV0z}d#IEuH&MzdyOmyz3%b#TUR-?xEc{#m|mzGD81{>5bRyzze)ycym| zRNij1<#;(!c}E^hH>^Vwr1|n#P=Y4AB3#p4S6zMGeOw9hUU7|GFblIq#FP0Qw(DF} zr5vYK3B?Ak_H zB^{}Z$8Nfv$cGndZxAVWsRxw~%75f`?U81Q)Q%yOVV-zadJaC%Cdn;FD*sb+xC&#< zzV6u;852`HHV}J0W_;vE*HtB_ddAfhq@xme5`4-5Pnn1g?7kO-Zv>M<%|pZd`_iVR zOi!KYO95T}3;#FiCDYEN-^>hUcFz2gIu}}Sw7EkpiQP8?+`idr5m#1O?k!|f(#;KH%X_LwF`}yND5aMp6Z3~1#g90zpq|`>~ z`2v}t*zn5W=-^Mr0{JVd4ZQW-!$=ieDz6~3s|peBlDrBe4GoxM(L5M4hP z+YuK~Pu?n@V?CQob?Hh{ckvJE*X$v-WHVM~m1xMdP}qmBwz$?R{EWJ(JG9p1nJU&) z>kI5{Wr$T7B>pB2Bx>=tYpr)d9}lH3Nt^H??oI!z09wC-Tlu3HG`M^hkWOKZ2~hxJ+(q+XS*jie|=<*m|3sg63rJ=n9+^S$>(L@4qAb#I$R&WPwA(Ilcx zL?v(+KSqp+`Z?N@ZE&_xG2Yl0IR?bl$?eOtId|QN{h>-9=70DntxrbZ#KTYTzbKKu z3cpvTfAoi2Nh1Qo%qm9TjEx_qz%bX??DJva5?*&JNni*Zqs>WNYIJGxk$wMMeM>O$X z_vVd^jr4l`ikBL;`N=3BE_Nl~y$`WyBcy@i8|v*WA=-AozQufOecNkp;(8}z6!xl1_E_S&mVlN21aId? z;et4f$ecKNIyeQn!A&=?MGZ7XYN9+eZjqlk6c0#W^8qMqi^ckQ3;ME7byG$6HTSok zmEPkK2P4NswTcc#zspuX=1|PF>?L!|&Uqu}A368Mei}VNUJ^W)bpDO>>c)#nFY3O? z|9(qaoA6+){x9&@wjgpxm2!h_am)Rgw^8&z**nM7jQmjmO9g z{}U>J|MWxfez0hGyq?qUEd8l?-A6pjsbsa*bId)}by!`fc$G|irN^bh>~vqTXZ#qx z4xUA=;3uJR;mV*NAGbujpiA+S^dk>(tIz?8)xZx^QU)zT>W1ZU0h;lK(7Nzf;rHP= znr6HNz3B)MPtT}GlZ^gWj40A__y(5XSxFLp#xEFU=QrmV{lOEi$=)oN-IL7C26*#J zNFeyzm59HJvWF1~wh)Avy2Obz03m8Fl>rX3Lo7f&#)F{DpCz{OHf=f&)_)AxUia~g zE)qY0vwWF2(r?LIO2BHo1{D4nyy%kzw{(v6BopkGnQ}Y%gVa%~Cso2%Sx;Jt7xO3b z7|SW6l<$?Vl;X;tL~ae18jut8kkxq^{>tmfWx<#V{=so`E;fl@i68tPD^5qwSWOP) z0%8GO#2?gi{8ZvyVlyk*(}*hCO=R13;~(Rkk&oI^ok0(9sy@}V{s*GlV0(z&jGVlU zpp8``4s07Tt{(BAF5hrvflq07` zG>&l4YTjFa21DuFQ&*)N`uOH!_q6rC!{KkOe&TNVH|0BJnKDzUri`Gv*JdfZY|53C zbJ7rDI2GB}ggb|{K#PDK_&ofL;R12%pmc?Jll^jjd8xF7`osr78~c&yh|Sa-*oO__ z53_)ATnmS9hdXG;^zr5(`>?Q**fK*pMpR#KkV1MB_w*?-POqrew~&a30Yv4su^O5E zjBcP0^raTm@8FxQAOfofG1j%k;UJs;V$Z{?*Od6IMvOpZaVs9Uoc0j>0K4!~_9rW^ zI$V4RJHa^O^PW?;LcxyFNLVQ}hIfj9u8|>rCGKU%y^(nCm)39OUk@docZhJ9N;0$Y z==s@sEW(HMnryvuL`jTBgC37h{xA{WcgT~h4WIphr|%5;n6;_Ek%v1?LqlyrUh6N^ z$*fN`jd$dCn&fegC$`b4<9ZkFoI~x8>+tk8vKJS_Pr1l=&SO^xJ8?XBOQzik)V4S( z{7RHS6{G&yUKPxB93JG-wJ%VF;ueLihtFK=eCH4dq(uv zMfv&6#U0aQ2PRvOnT79H0rcm<($79zb~zt z2d>--A|&S%QMQxuG>O5WN)dUs`$0;zYM3T+{@=7|8=sr`+|P3#vDbM8?R8>W^Wz=tYkRHHWh5v}R*`OOp8u9v;_z0k!+n&dH1z5!>)lKd}D$;Nk8BpY+5_U4W|3IcS*< z4V@~p=V{9>UU%rNO5M^#`df@P)Tf%q4{*+G;vl=y@22o$bDk1GJ0C*Hqg3v6DmG7q zb9cducd2`!(T{ZAMpo_C>-_4p_Af?Z4=rB{$NkAwJE87-B89f`F&^5# z=T(s2l!XE{nbqV1LD6)w9X@hT=l%lHwiL4>h8$UwNTy=6Y5+X_gD?Y0GKqM^v3#$= zIVHJEG#ojW@tuH|zdQJ#^|+%(<(>PCk-{@N;1~FfnldZMAlgn(r@o?!<5HG8b%eIn$U&_@OFdZ?Umrukk6B|6LLg7d!MTNr*6KjS zcqekce;{AA5|n=o1$a&pYeSuu>c zk2by`TJr&8^o)@O@&NRQAiq1bb}HaJgI5(W)`P*Y5J~wXwEK*lkv6ozsdL?& z|As@G4shE4n4d+Vt5a1-V4O|n&N*hnTH+hVBH>3NOUEHqzecJK2ic-EcC3oXxWZJ3 zsL0x19*w6G8S$SYuSXJf*$e$*BGPFVSO3E2GUm23#;0kqQ;q!=62PgA{De_N=oXmQ2ACA3?+;hhADZeEUrF)23 zz8}PZ<fHR+i&b$dGj-JYZCqj!A zSycFgT!m$yNfZnm_L0$g58tIx!96crS(Ds%r#5(9p3@PI zF3Aka2NyZDs0-8nX3V&r>ij;Xv z`~HR7_Cn`l$b+58&GU@NMWn-j{IZ*G7vS?feA~m3W3=uN>+0|PzY!U72^t>(y)}^< zR8F3hjjmpjHL5TYtva7=c*QXbV|Y2$;3N5rqTR)5Z+qI-fwiPF=M+OWM{>?baBoc{ zX&k+ZKw3C;whPhbs?5w{(6%&3O3;f)o}^+2tjhm+ke#*RwW`djXk>&Rs@;Nm7m;9L zXs9sm$;h)bxG8~kazFg=4{ct<|9{f^y{zk7dGFzvg8}OBOFVZ-%UTni(LiHlg><;= zH%4s>y-tLOojP0j;hnGHfp1w6yRu5QCXVZ8Xg>zIKOg>?0r#wcpZ1Y2bpg%dIVc>v zsgJf2$@o7;;Y(=cR56c(UIL#kBwz%+&W%*9$Zu8A3XamUNn}>^WbRi$=kI_#=!-VF zoK~+#=UPsy`twT%j&(vVG(&6aidIw-xlkNk@N4GDWZM5XYv(iib&9cDh>RFX9NJL0 zZyGaaA{5z9uQx)q15jZnI?n}YmcrP-gN7d&`{T^b>wMRkjW@W<8NNS;5>JR5a>4aw z7>T^ha7V{1MsMyj=N>}e-EhqVcRu-Q}`f1qgsabr7TxElD;f7 zaQLVqzg2@v+OdLlWc($x(#vq)KeTv0EuO_(U&Uyygg$e#=J{@9x--UmviNvE)S1Ve z+Ru9%BY2Mogqx4(-6WBihjd9_@pD z@f-N8DDD{F4{rLQno;rF)3>=?~CN8r^5k1D*D8nUnc;)42Wv-|Sk2uHE7?*pD z#2#9Fn%19y-a7S^sxu$TF~2|IUd~uJa{duB-Op$_tKDr@=qt1$h8dQdzO-OGsv{S3 zz<-&H=_w>HD1uz^f|lmz>9v@3wdr?f#-k&2Xh2IFadk0dNOrC)z`Gsta1k=}7&%vd z6pa)r%s)UY-ok$42dFkQOBQuwb(p}JGmxbFpw~^V*v=W_INk$FlwfssD%;0`Fzf(z<51j^j0t@rjc>9l(FB3#JbDaIeO4F`0N(Exfkx+44?c#uGvuX z3HmS#MlkyGkhtqO>wXp|=YmS5=s^Y}bBQ(R4m0E(BcH@xCytSAPft73vjMcPC*0Z< zSy~+)E(KrIWJH?qts1}AfZh&GDniRL$o~?&opDQsvtGfqx0%J4n1P#+I9r)v>zHE) zm}3`d@t@4P<+N=qzfNLTGXxEI7;}0mYyVz2dl7@IeIgDi>p1JWIO-T7?z&B0O`BJ10Pujg0yc zz8z-#-$KO~&_BqXGU1wC+})YU2RM2TS#SbAy1-c`GoU7o*u>XJbz)BIC6z`Z;sj zsf2iik#_X@G@h$6b8m9=B;0y1t5qjC;}ku)2}PW0i4SO{Q@`N^J$u8`cJte1_~S4= zxyU(AUw1>v_0W3_GIbMww<9g~K;ga6d2beN)}V_mgO_F?amFDzr?C>Oq=y%H#vSHe z8nY|R>gTN2dX_G88|}u~6Fh}J0p3COvPI#MLX1Xj_^LQ-SV4Hl(Hfoov6o|QkOt1G z*nw9W*65mi)`j1z@V7Sa!pt`>&v3EN3GhrWw9JVVD8QZLkd6Xe>_R5kRGUoU`3|nU zGd_M+-7tD;PR`0j3-U6{V|lK|v8Pbu41IVFO~cUL(LeLRBQ?-!KBGMa&=wqykL6o5 z?-Cp<$2oDdD;I65#;Y=|Yk^#>#BVu}I5Z%O8X2@fVYHonOJ1aVJ?>MJv$AteiL6!L z#T-e3f6p))*BSknjF*Ed^$4zb0ayRa=xm4UogMK^=Ez^X=P^qA86_f6IpY{3yO(Qk zbHcQD6JbM@scsj!DJe+-Xo<4(sI@x$=PG5)_ypPe=C2Hf(**@wNwk{~f_98D#YK3bgZV>q^uj8vKrN6|ZIMEboNCH>g^*9qzP1!nIxdUr+}y(%p%S#jsZHwa z_8g7i1hjhum7LvM2Gn@QNI0wdV|wRM8<~Zn0-I+VJ=xYOX!H%p@O_1G{;8DlI zQWXBp$2&Xn(9wj8Gmmm*abqs%kVLDT&kS1S@UM$i<`KQS4PU;48t-{K`obI9mrg%I z*kh7eK@IxvXqj*6wZkuATAaiPhMGbY?M2!a6;4 zeksMLv-@^*N-wR6gdYp@TMX~AJi}yv5dmMM(F4amkqgW>9~Ad<*GS~6$kk!wu_vq5 z|7U@6u9P`57rg3+O84pSO~&d5^X&<*Ot{369U zcPVC)QxzizyyNJUd7)}qC|HPgMljkoSap&7?&MhHg2ol#nzH;}AWOn#hX$q5VoK28 zy!6%43ZtQQi7Y$CJNjfpcid-;{s*3Rfq9<-Z(LyI`InwLJL@y_a~mu6HssGj)`-=# zdlNg0)x7>@J&Z?N*~tHYvX-jszAi&JKp2gXYetnZU>16Xzxdh&hf?GsrVlyv_!SCFN7Ae(AqZfZ3n+D;R-5^Wbu)+ z^F2h%Uc*Ue>4k%ukw~j<@XY^ckuy7}Bg*>Z*r~Gfe!=la^j3fuli`Cfvc^Swb?jLl z=0H*AUmQQhIOw1oK}l>b5`xH=&n841I%XUHw|sEHG73`(dmb?t~(ljH+G2a(1W{k zRs$?*Ezsl3@-9qEB4~@3JATTOsRT#s9nN#nYKh(k=(n4*igSjev3uyXV`nJH^K;PJ z68u_#Yn`>DAm2UQ&C&HNM$Xwwe!{mR^vKcvT=X}B_Lt(E%JBdHQRCArIysg;7q3bj zcc@j4HN7&k#IYfk;4a1ZCURy;+UKHO0#C|`Omei4Xr7y!y9qoa7yqlstb$OhGVQ7d zEo$=F8BM}L>>SSjU$P7Mmi(Uk?&`j8IDzaC+ojsi9QGKY+}WK3Wr_F`t3B_eGQ)f zk9(!SHBni2*5QE=6f)t2-1IdM?P*B=TcGDRq_wr-oM!ZrYquQkv^(LsuJZ=sj(J24}s5zZ{+M6*J1Qeo z;0~@lgPb@EMNaY@fw4FEboQ8Wa7!b`^>g~sgB~&v({<|@ZpQJxWtX6&7vKpOoyNe@oZ_Pxlof0^s+_vUA`{jjxi%mdR%Y$qr}Dq^`%&7xnQQ*$ucMLNLvlHCD-)C-8+qjLXelV(0M4ii zT`MwLEosG9wBD&F{{{2wJKE^jm7Kk1J4Wym#;zv%f-@e?8O`=tJI&_Yr4!UB2Q7** z!z?6*m9=WWMgAC!le3=dS(3n!@YiVD9(c&vi=9LYpFp}h`w5579x#&5T5t!s^?)xY?q<`efT?x zJ|)w_=d|WB-!H=zPZ_HaG`^1HPvPv>aHF#;I0oljqdzC<#T$Bajd_>JQAZ|w(3}+H zQf8J_!+=J)X<;!qy*j*J3_2I)U4w6qC89W7;dp|a3=>Dca(r12`FzI{uR>Qx6TXXt zPvU7OXw_Su^pba&wdyhNBed;W)|#~o%I{*f?_f6XU{0K36t=K$TMp+O=IvO14srfb zzCGoN|KOjq^v6bmzF=hE^GxS^9%$8wmX(9v#n2erBcXf4_g(O<_QWclY95A5tTl#i`9qS>a z)lxWq1+Ay55PY+odAfn`laNjm`M!i-7IS1Nz1Yj~bx1JB26qK2+=SL5d#*~fwFo@Y z6e-k-R(8hLIF4t}<8v$a>9bf~4q#8Yhpq7)k#KvdFYn+H#S7O!CVIkuFNs=t1P<;? z@Go9->^1)9YgAiWicQ<`T+YB&xfmJ@r&j}@%a>U_>Hu%Hfd`AzuLx-8>}ebgFNLuo zl7jhgB}=m2WmG}tX6!yPVvY{&p~Ycz>`KtY*_nI}b$ig-u}GI*JY^!({T)g=UZS7q z(MaAi`DH2=($)N3MXR=gX1oKh_ewq&^ZN<>D!b|132g8yv9oQ(;^z1w7SiTv9N)=E zId_l^m%;5L*%m;@$R*_7;XF`P_JnL|mo7tAkTnND3Zg}0XTbE}YtPQn3MfS8|9(1Mm zooT@r$fl9B#>o=y&*wz??|2y2(A$O3e;yQA&1(&#vJKDC2rP-?ICBUT>?$8e(qjHTo?`WmRnfaHQSC7!7oOR$fqjjF~dV>^r#JS0g zT9E4<=@82bkqar}SRYMj5u|VFjAIggb$nB`=uL6Xh~@3*N{)5Nu{t>ZKF9u%!Hyx3 zx$NgUm9_3ao_UG2_ARsE84~3t?fIAAuJip6ZFE*RN6PGF_9pOopLullL=2_t3t>%*&I=m#rMVn6>`g=6Q~OFMxaJXxH!H zjk`#g7f|sOPrF9z9opVNvp)e6bPu%GV z+?2*=I+E70B>17&OYZy<`ak6==kI$)$;+&fd$5JvK4K<3@%dseE zjK9qeC7qYfahtoo=iZ6*{VOr+!%R&C`LOPwHM`xhN zRnCitRtdcBbL2vng=`yRvxBQ{ar`3ZU8JoJ?>RODrv)MSOJGOlhMOF(b0oYPLWe7j yB+H(qoocLv61?Q>Kpo4LvzK(N2}xNiPAdJ9STCHNhI7ZXEIyRrx^&Ku7XBY><(=~Y literal 0 HcmV?d00001 diff --git a/tests/assets/minicpmo_4_5/soft_interrupt_16k.wav b/tests/assets/minicpmo_4_5/soft_interrupt_16k.wav new file mode 100644 index 0000000000000000000000000000000000000000..41b399af0377014c6a9ccccc4220d0629f4288aa GIT binary patch literal 820780 zcmeFZ1$z`p&;UF>JG1V)5fVs%1Pku&?s|s@cXv79a5R+`6sx?rpfocs@YoJ;K)f%YQK(z*{HBhaAY7JCtpjrdf8mQJl zwFas+P_2P#4ODBOS_9P@sMbKW2C6kst$}I{RBNDG1JxR+)3bGeG~B6dt<&@`L|Z>VM0HR<`HA_58Q)s@8ch zRpDprt9VqUJTyPADmgruA;1>@^JD{dvdEv zRl!v>9_+uZ!GR{K=>FyJ;q0No|GBETc~U=R_VBGrc}n5`Q_D}N>h6hgARh%TvJxWy zq5c;~RpRMN3J=wPLsi~SjEBq5%L7qRRjGe*RoE(ePrfHq;bi|w{mierd*u5WJktJK zuBtXq-cL^d(s>|H$)7E(O3{_LsuDl*JQ7rC!Go`Q9-dl$;;J}!;;Lu-Q$rri&$6CU zRc)z~;AfeibyU^vq4tET7X6g&XWDc3*ox==Z+rf2y(hQI0;`@q1nB;?$e%Xp!TkLF zw4(p^XHT#41V-dQ5WvM$2G8AtVMJGDaEOjd*a1>@0NcfKa9P3iGxlITd7j{*{+H7F zPr3?QMd2w`{!g$1_-CrL66c9aD?`=2ypqCGXQ@~$7J^U=q=6s@xaW(RVmi1yak`id zf(2kL0H0ln&8~!UL7DtYD8I6Fv6%Z$seC}4TbY*&?w;}QN4>1DJq=1m{4>k@!(E#s>zkz!kp!!LX0#Y6h9{Q@-Q!X8ptfH#o zngDQ_l{im1PmV{9IKVFnNa?9P7vy`&d!+Wr>7nzqI=`}Y>6Q5&&3I(4YOg2G2C%24 zxs`e<1X{IJ29H)fkVjKh`Bkx}rhI_+)K+zQ@E#5Sr1farlk!}p;PQm>fAs93@jU5L zaQ9rrATF(Z-kwxBNaupIhr$CDf*d=5Jha6Cc2v@OET_tDJlLvKmDPA`q8!+r$9}5p z!()9_c~utc$@^)qc0lQ|Z$~Ay#|Ep)d3bxEDtoR<*@1m=mGqu69-bcSc7b=A=iMVz z%2S2+w8&c71`k!$<>@<~a#g*@L-(@R0uysEYsX_diF9symw`u zC&z>JgMun4g8+3cs1D#@tfces@`vh!+$v6i;9d&~23HWkdMG{oJvB6hLO|+gX$_Fg zzp`vCAd`pITq%_=pa}-lRT6q+Hdf|(q|$*a7^H&0JqYj%f|^uP)(345ftmw;bt)<9 z|09z}iiUuuIVfcYaDBk7A%N>t^6r~sxui1h&97Q}v_bYP|Ao_2V+dQx?O#0rqt5Kz|v*gybz^wJRI2Lo%B zRnjzv#y}bHAh-`0L0W@67}5Y*97+*&P%luMKj2XdSe!+~D<$;>H0h!@Fr(JcC?I_x zkiH=(LqQ_|Z+~bUfJrb0cLI`lFqvXEAcr@|uLI;zfl?@xE%tzhLG1yx44Mo@09)<| zWa6PQM_;ioyhpqbTGbx%{MZ67D7HAyUkSUI;6oQ&0=xrKd&W6R4TE1bQZ}7NUSO^TjJ*G-@PXf&#bSV z1)c)h-v^p19D#=kW8nxv1??7BKrGaS9}j;PJ3@O!x9}VuE6jyyArELv1}%b$gb&aa zp(l(9N;FXH%TGW)@x^Ey_f=HD2_R>NmEa=Hme3j4&TeEBUkI$&B7B0(n9;oz83cC~CPU$HNA`;( z#8rs8u#S|R*DLUfyx9(cnseHy`dAN6@sD0 zf}JenmpXUgIwVxk^Y@8<%qDa__bcYbw-DM8H=RrH@4 zcK?FJaK9iY1PyD)Ym$qZr}htoA9mM0fu3#aLv9uJa38Sdu!L{M9j3dn=LL&3{O z6PYfo#iBd4KV~d!mY=G5!&_|dVo>bduJ!C z6E>Q6Kts9O^b9sdca`lJ_nPlJ3lgT-ZHkBiRnWQn3R%hW8Pp;?up&G(%rY-i34M27h1FTohVXZ|$5d~A4O^4caU}@J)({|aigOG$kH~}D+YY%+ z#0}!18@w^a!B8mtmoSouBzp7th|6_D!MX$a&EyS-9-q%06jfYvc(E{*Z;5T<`omwj zp+N6p&Ur*V;VZNsIsku#Y73!YJncx-fkwOMA<@Wc;uYh^ennRjzd3KQv!sn2`_ZOM zAyvd*;x^(N_;Erv+)&yLR#RTc7N(!$ndAlgE4CHb(gu#iZrKX((ZmoqkPj87%EuSr z&=Gm8(2?uGh9VoqAZ)E8oq8oswy|&!l#6Z_C%XGf*4jhZQ`Bs%4wv9sBqS0jYfp=Ls>6|K)zxl*#^Q^G@n`SxP;2!VfaA@<5*1B zlk{=k5aq}~_9)cUJr3I?IaaRcOXRPRCGJt!F+ybL&`puG5GTxHuaS|&YIjq25o{qs z1#LwqItLzAz82^+jaVy)>@;+4-$8nVD2qBo1Y{~ z@Rs~!cp^LzdWEzQ_aisOBa$6(kgEaGgX}?mc65O|N)qweP-CJuEHEdiY1r?~4Pi2$ z!#QL%2}^mJ)sL*p)Me~o{OiG8=YD6xD|{Fyvr;%q9z$wa{3_lN>F8}{h?^E7Tw3=V zxB(gLI_BOh<_c<}ku%59RGv#8vk!I25_I5I-V)NXB16@4^Fim9@32 zr+c)swY|0Na5+-4#CE)FK*6KJ+vU40!PzHrG78QXj?dnn7hW=;q<6u}g6FPZS&eH; z`3SBJIT70hmr56FCMa90KdHBwKlv>7!D^kabtB-C-;=;IK^ip8e(lJ?l%G~^pTuTUdK|&jRE!9-9P^U9R8=u#Z)VWb7 z)NiZ*;d+iH-y7|#xz4M*->^FU>TavqN_$x)H*_;~HS+4gk}bq-#bvpa#;L!l@6>6c z0DDM1BW2hRBm_P!TG&8$M@LHO>>?y@cg~~aj2VeUw0aB?($d&!+0)C*W`BTa17| zRHPQ|DQ!^lICFPQkLW4Uy&{U<5Bf0nN0<0X?=Lw`(bw}0^alEwUwe_ciK)cQ7`tjas!+9E8jkx*7}*F_bID~bRa)4>$jxjNbXDVyS3qa(n1qdFCYJ%8os`u zMn>Q#@hy^AzbwS|+(tOo+jAI!PV;N*w_CYdair$a#&a94syjF6M!iiz>48mx#sp{j zSqvKuzxuTBex|LhP|GoeoKBbAQ?yXrAlDH_`Et!Qjwq)$Hk+2fp6ByZCi)l5x^q9#3$jHFu14Bran*j5huj9jI8xF|J&~g zJEJn+G9Shy@j0b&?>={m_Oc9fggJa1TU|An+eCfy^cp9%7pXP6KZ5TxZW6S~FU@a$ zKx~~AL7Qr$W~p+%R_=GyKUX!K@RioqHqvyWlO;!Fh_p5}P{yc2)Q{=AlKGMkG$ehE ze}a#2G0q5EYWb9+4OvSQzr@iot$t)j9E!sHlTAjQ$vC*Hq#I&(IbL@96 za+sX8-6wF1A=Rg!cE04Ta(BRiMt6f_d?uRT`$_A*uI=(WY#gR3Q7<*l@X;IQ%BDz0 z$$!x(R83?iX)OIw+D^VsVt|@GpJKNcU*-NSoPrM8Ci)GtX4g6{}R5u*t z=V$cunOVm|wl%v_+ zMe`hi+#xQ4VT4Hff%>zyyEa9G8GHK%2GX^M```B6;w$=cerIcp^j>WkYv^EJXzHoI zr#Y%Upp$BUC=SRsD@~gBs!Zu<`XY@1E2)o{AcbNR?v87iOw| z(vOJ5^kDvtu`%vuj!C7d1yHVU_PosX=|j_8X(?%=Q;HI~#9hgCQ?94C%;}LAkayqW zXI1yaN=zMiObT@Uq zbPv_fR3|kDG(*&*RPR+klw;+iWc#II^muAHF$uXPN(GJyaM;Q=7fmf#m-iv(Rp!); z4w;wJUjvq{n=gd`E6LTI~6lGh=CtHs?e=u+O-biN>rUudD72h?YZnu87 z;hEugLtA4#Q@(MY@x7_8m(64_4l=yc=jzVr^7IbfL9Le-(bmwMR?brBWubI!sw24> zTL$?FyP0*4dKJ%$mlZV4?VlZ#Wy$!KzAWWT!t{io#EgU;@d-(H(l%yX&Hj`xD?V1z zy<(#y*)^9H#BA&l*`1!J@Yl@H5yqRQ4(3ay-_3fjKg?cUR&?EUr1w4-n2Tb2qg|l|3p-&zIy*%Bq`@n%*{Dn%XGw zTl}uL1+l^L?UNs+iD?b9&gR{*6c_C)dtmG8800=F+(8)3ThdwYUyO|K^2+kBS!boB1vwZmL!K4q4eD@<2Rf^o1>)YsQP)f`t1QWQ(KQ1ghRXtwx` zo$S0;;a9TU!sHIjzMe5JLy@jZ7UPGL*Z1OP+UVVJJd(ZQ(?LE`0r+JF`lewLFvWYT1 zG)^~k(tp!#)D$Xx75ePW(Q z?~W}^Je1rzP>tf3Cq-!%Ce>luU1!_22l zh354>EqqeEHhCZQE;J7{d7Bq_Wt!HRR++|_RvMb?&H7l4ziPewn)J1#HExHULK$<% zak+eLQ9=IboZ6Y=Gb9-YQ@O+k@o@aN=>9RA<69)HPT84OBfD9yHSb*E^|C-~nEeN% z5^;DB`dTtsW|PlWm+D&UKj|8P=b+cG)!#8XQ-O3FHt{znR#ln@^sH$EXR z`DE(g^zy7(d0qu?ERTy{l^?Rba@x3OPzPiLhEmUE(-k9B4YWsfD%}*_3jLq@V%-PB z71Jkks%fS1k#U&821ba1rWn&q( zP#jv&I%jQm-|WR%Xu3~Ilca(1`{Pc@WWTo<(zfJTg{z=hoXFmCY?4RIn^3Gs~K#$RaY5Q|yU#6K=+3$G=Rv zk$NsYJTp9}Q-NQhy5w5LMMtbVhl_yTqb5>KOXLHTtSUpR(ht-Bt*>F|2WAg%jAu-H zO}OcWah$QQ;kyAfy)^lm4j7snUhDd3L$n3zRjL3bEpJD+B8TCL$WtMnd1?Q%Jg;a+ z{^gwgnd35BWT`V#Q>a8XK@p!BcR%57N{@7N+PKVSc@qjZ7KfEza{j?w=8g#n8ip^U zAO)rRUA^k(Eo9H&%78IY(8tmiik;mGPJXG7dGi0G6pT4mWKz zg&C#B{<>1#9;3h4JFhl|+xj8;CHhj`YQ0Gtp?obLAZE-MA4G+C)UZ%Il>?~3ciYukx$h; zS4F6zwL6XT%&oj}b3=W1?JMI*uY-mLT8CBW{EB5ob8>t#{zy5TSR>{{%(>)2iA{gl!>7buPxnm@ zirpN4JUu0^qDU-nZi}*Y;g?`Thy&Po%A(n=ucyt?Sd3G>PIx^ui~w({nTEwCSl3)- zQf8|TYPIV7%9F}y1)({jzM{RQ{j9yIJEj?@wP-W6-PCIp;}wW9PWC~PfD(el9b=P~ zA1zd5|C!o6AvC64!syfvsi_gtu&yz6Gt$ytM$L@Ak{X=fq=+nSWbJMDanFQ9iNRzk zSxbIV6Rqi}+hv;TbJr_a59zuZ73RMB1m$vtsLE1zQ)eq%GucB6#15qpyM%csTxoj?%p`9MuYjQ9j#ij4_*adc$~g zV>74SB_Aa3rJSgislDVHiIlh?ou@gex+GQ7CdC%@PE9}UYjs!Uapif9P8XuyDhs2N zsXR$AzFi#XHd_ajyt53iRz5gEOXX&iahd$TpF8GgcJ(7Y!@w zUVgz6`!I!f8KA zF!ho4mmijXr&6gn=?&#WNAN<~eCvQVYu)Q#I+D(wCC&h?z~Wy|${$;i6sH zgVPQsQgO?YTV$sRG6l|eH{2`Sb{%jl!hy^@V!U729Iy9=$l5{YFa0Qzl)a<-Qr)Oo(n49H{4cpnY6R;KyEUye8S)F# zU9#iKABty^yV!E1Ig%o@bjfUrGDl%z{;llpDM7K#lDg*A&o~pofBr4$ME=q=Nrd*t z!;H*AHlN7-&2qMUI2h+T3UkFX;4|#Gs=j=&YLjd7rwI=7Mpj zaj^D-JVlyBPn2iL^QC*p&14=`LG6$fQuC#YsBV(c^an+pvL|>4x0ZKS)>onO&eDaF z!(=#q4s}2)nK;|NvXIhNCH4aEv^{aX5+~&D%$3Cle_a)IB>PTot%NQSiumU_XDz?w z{gpek@VGU^HJJT{UkSsKzOqiT#`5vHDBlgfZ;W1sNUy22`ucA&{i$uJziqB%TB?mz zE|f*d!W8qAb);vB->Ft~H2H=|kj#)qlY{U(#CUp>JWwV|K1r@f*UCr8CQ^sVP`oF8 z03*>v{*G&+J;c_hqG|Ed{Bv1nlFp?3o|~0>D6a79_;@m}e$LR?fFEMgn!JL7McJz} zy)2fBd`GeKhC54GM{J?h^gZcy{csyTM>* zjNi&x+-K}(D^8SjD?DlWp1V8cwQx+ZH((_UA@n4k$}h_&&^I)DeBb)DGpFkpcy$ek4v6td*Dun~@Lu6N-x#dEr}(JK zP^T$2N|#Xa^cY!NNjqW(UW>}0M&TdP=lDa?Az4Wj;B$!mgo^4bapNcOSi}#F1?zrb zCx;v8E_Y6M=wJ1=Jq%WD{Y&l&$RwD5UU^GMC%-bT|9!wIj>-UAF4 z%~M5+CQ6khA1*V{%cYa4w%B?6sw9?VP!04aqL7%#6<8`%2tCBtN_hM`(i$297vLN4 zzu{D_F8^3)DnxMcj*WJA#ife5K+;A9wF-!wG3kf1R^$@}z0>Wnf2G#W`D}Ta{YPq6 z=I^;V#m&ohmC7shuJ&v_rjam#^it%|^Qf2d7~M>tW9GXCe^XPhJfAtBgI z+xSwKu3oR!YV6YC^m?j}bQGl{7h=WGTBI}blxT!%!~mha_z_)!wuh`@LwFg!32lsw zhVQ~@=pj4}eI>5smG~&=3j3PP6};Iw&W<*9#ZlY1vQ{NO3J2!}$Ae z8BGxmNHF3%aR8quti@i!)37tpBaVknY$U&%-|SRy_nr5wIJBaSD;aFt?0i;orugsT zEhR%rj^_mxeJUQGUs1HVs6pvz%d&!F<=yQyoeN6NmaJuZ3Om_4tc^b}BqBBNYs6(q zg4874L5-04Nu7$}nnmgY*+n@aZ!Eu~x}op_^M}LqHyKYwQO~4TXi9RBIFCQ33A&lY zj{4x~_!vADuZNDM?O>lW7VioBAhqGQm_IZ`2o=x6*ZEjp#(m&^ND7X(@_)hW?Fz1S3E(RGYevy@L^=3HAvO!Q!BBY6Tp^8Ra2xDm)Jf zhAiS0bQd1xJWWhxn!D~0!0^}`{1bLHKa%ZYs|7a|d~E|+KX#@2jpH-3&D~85bRMi& zDE4x81*xXOE2o>`te5#E+!J;sAMJR^FJ|l5EJ$b6+QmCewiw3_6kEGmv9pMi>^kQow44d!vv{dcgvi{j&^8RtXA#HT zw~@L+x^o;h2wv^pj#$M_u2QJC@SUHAS(w$#O1u&Ch|S`+Nk;Gvmqk3rzs8he6nxt) zAdeuk`xJGJ{V1kFZTNb`SpHA2_ufQujeR7Z6t|))aUE9%AAsM19h$?mkNpI3016;Y z$TLTZGOje13{ZMo2a;jpNHGTEVqDQ^`YAGoOTzY5 z)W>y_X)Xu!0Z(!a2T$xed=u+g0aj|k{@+x1y?Z&CAa-};(ogN)*kW`!vkdRzUI$iX zFzx)Z9b7|dsB0JW5bTV(-Q9&7VxagpHv%8Snqe3oz#IaShY3M!M`Q-ujtCbX3VQs! zV-30jJ_jELer{#383sG7@O7%FM4?{jy2(wJjj(^iYKj+O6LS`fwHDz7ag#mH)dai6 zFic{HpcHmGY=Z_M^}*lY2SY;`e|bvDDfw&mhJ#?2di1!Fuc*Fcb9J0NU zhugEEP0|{+RIE4i9BU;Ggxavb;6;2f_f{MTf5J0e31C;Im9PYpIEKkbS{Jf372&*v z7V&8;DQVmO+x^m&0^^Fu zFR-f_RCbMZ3syvQokSl%VQ>xpcm8kcY{^xMf;6lby#ee=t>h<&cd^6lX+pZFjv#lvTy3(zrsI{t`*7=oV30Z$2R2{(y7K|OO=Xg_hg(~Vyd+riz~bL0o% zqZo(|U>@Ucp$Fz!#ZGg$nR)WGiun>P7(Zqp7DVq(hg(Q5+sXx(Oe~v>x5K}HxA{K% zPFm+mBl~iXL^(Q$zs23( zQgy7~;Zu@y`)tXd?pH{AlC~=mInh;IC!S_sLd`Lw>p9*L>fo+HaKM-6i2DpajK6|J zB7OOLY&y11h~nR4UByQ328fL+ckFV9(Q(exqM5?WexaWUhumwV-fSRy7mwmLihYQU z?r3H=J=k?noI_4>-oJ7Sq;k|l62>p9#&)+%?%KMEw<6dxiS7Hp6gqIJ(e z^N}yE0l*8R0a8ym%JmWRk<)Bf;VpW|1xxn1OYo`u0P&eP0KF}IgChBCAZt^8DLND) zT~YEo)(`M|)XA44U4;%fZZ*p1yXp$xunq{puV(h)7x@%qn|-;ose3eX0{X?BL$_h% zVEo#IWie9FJC@j|N!PHw%RcEJai!Kxq><8;-K9UlJxgZlrm=o5fjEiOW*UJ0*v?&- z(6}y$k;Djo4q|h$k{zx?_;g+X{!32yftY1~LF=Kbt{O;NOvgSE%~&h8BNTxhb{63c z;YK3qaKd=ZjO}%@D*+r#6WrTsM_(N|tf7Ilz*F z-Xn%N%#veFf(w?d=Q~=}Y6E-3o{p~o9y9@HI{MU6LUbm)OMf$lIpE@Gzf5b_itXA7 z&SkC!%0&BerV~BW#lwY?^9~vRfl3fNz=vD{{ll>UtoCMU6(yCN1lJJAaSoQsyOb+2~~qvAN6W5hA|uV6&>65A-vWe|E;%G*}r{o&)bcuGQ) zR74U%*nZ$6a~jDxd^uU;B7SAyL{pz+rHEvVv<#ZE(MHkC)wW)+E}|BVdXB15(NzWG^BU&^G*2 zz90J<9%}0)9w7&^v&3v}CWrG36aigBj)U*GP(B7a2&a<3~JL8T59>8jNB0q~c3+!(q_l1{>BKwJ7Ej)CO6xIo}`x3NN zxZs>54C8JycHU2vFg4(1&^F-vNpU-ohA;&!MMmK_v1M>ExC1+SpR1Tl9J*j8v zkIXfN%S#FiC)#j)CDk3jf)>l3*Z4EQ+xvoXvgwHVu+~dqSIkljP^2jaN}3~cfKOs) zx}mI><@TpcWbO}&~r@^*LJ4De$=|Rw0Tiz@xs#Ug}VyrVlH=fc4B(Hq}d5i69&ZIi5eDh z{>$EvUEVDZA7BZWe>BK77qn5P60Z(kQvWLr>bJ}fsb901Y61L_`QVC@@4;dt369jVEhQIAW>f@2x791X%|1`PioAThc9?=pl=qbyJ-mO@j-)3-Huo;4#65>; zCt%16REl(F=R4;*8@lC8XSOjTIF#0eQe9DbUV46Vae2w{A~HWIvuWzp_{Wh6U)fK! zUSGJ^@KJ}H=Jgg1H}reZxGXrU;Tyj{Xdj}pacRI)^JV^S`s(nbA6I{qq+BW7#dT8u z>3iNQM79I=BDWhx2P_E=uIC%D%``&N*WNmRY>u_?Ma2i#3r5ZOxfs`4t{-|vZZ>7q zd>?cun65p;d!X*Lrn`Ecv>j21k-$m{?6b>zS^su(;vxK3X_Wk&v@z9yoQW+G#&P$A zt>RPn%JQK_UkjfU#ac2f;g$)xzovJI{~WnE%=E$Q^`Ixk_l~{3&*t?^pSgZW^A>E& zscpl3YdDgN8zJ?S4`Ex;lH`ux(>?@z85H{=dxZOx-oJ@o>kjqCdU;mFmzX!{XX=~= znSK(z)S=B;o;o%|%)U|3(fTV?KwYE*=yh~!Ssk$cxY|6nrhlFDH4htK%LRe}p0>@P ziP#Hb1Fi*c?V4<{bDO&_6oogYY_dzr5P3L$0vab?=X2Q+?(y!n&ZE{wC2#Xf!9L2G z#D;ODKOTp#|7?2y;q{#7?5pEx9cpbG|8>sdVTU>|?9?t~v-l!P8~sN~4Bx4Id`9b} zAqkIT^f7RXxv(LgQrEBB)gEJ8J_;_Yb;aNhkFwdFr{Lq5MYOrbl*Q$*%F4_N%$r;~ ziH(+)dwuj<=T%4hOk>p!RIB8I;;><_&s)=Vxd2vIGWeUEojt>K!lcrj(i6C^c#SP( z8$ye*vDAB6sj`=HvScp5+CIEuXxZShpt5mgw~N#BZ>9B*YX9-$i^%5}UiN&2z9Pf6 zmG5r!ef-x+EBpF)S=a4w{b=joU%N%ED6D7cnpm3Hspxrmcy3Y3>I}536|ub5zLx#k zFK)M@!|s*`y=u7KmQQ61t^G>WC2!pE$UR|E#lrkGc{#-b^ICFTd(`_>U}1w!!SUu3 zRBNmQF2gtRJqA(eS1!5G~ablth1u7AAvX-w+y&tRr8mE&>0AE%HFz3g$O2oNH`mYpQLKqtv>zWKv%5j9c;6?*~5wzPH9dJra_fxM@#mt(Wld$a2z?xPZBJM)v730C zAoQyOj@Ktb;_45nWz=;ffRAO-oZP9F9@acJBD_O}lewe|Jt)lON(37oD_^IbW4!6z z-cRSVRPJ$31l29frGh86&s4Q zmgo5oGriO33^Ze0;=S*CziPrBeR}YzAZZKoSDpE-U$tA@ZC2-BgS>?IS;v!iCLM@= z8C{+m!oEe93+-9EJCWJU_7QiJJGFCa9j^breptXApYHm;lAF%S#lMwK18YvDt`H~& zj{!&gCLlc#7IMOCiOsTh>h{L|z8b%WUM&nREv_7e#{&QU@pwPrC-xWeS~8GqBXno> za8j^7qhib8dBkYM%xAK>&NjtOGg_po(#*-h(L*ARMvaS2h&>hmC2yp#-7u-vg4#<1 zvIBS@3)!r!MV`7it?*9Pm8`bTG$IV$?;P)1j&#B?u@jg{pDFt@6}n?rIzi;q)%)k(na&pL0)2iD(|hLYV$8NLk; zg$LoW#0&H&m`_f3kFZTE{k_;-Y%W_;W+)YldKIm)Ow2x<+CC{H8A=?N)H@@z;w&mv zQmUq!zYMq4HfVaewIbB9$J)G@bMzw?OO;4%eww&lZ0hdqC}+Q7I$1;23f(b7ZR1hx zpURWuAMg>;54E7XK`g7nTUip?oZNZ=N=HXGU70G2vR=miVJd9 zO_FS>bQskTN6=E#jgKc9khQ2j^lIuexkNIM>JHYpKPcvC^0l=7hau6>RI^#OQ5s1< zBU0gA@DO+#Kg6Bx+UELXD=rHwi79%WpPH@9)TCo6aq&H(l78Hb?wT+;&9^KNt&sW1 zKPuMgUdyjS$6ZQcK6l5q(fURx!5!jKrW=?|uLb@ZpP9XAK4B-%$O3gOfM3ZZ?OE*{ zEv?zD?4vlSfK>yO<7G|hHqu-=jaY_ZL^3r^`dZpheqFUk8=>8&nWq`1$yGj*&LWb4 zAAAAt!{j>ZI@0aY){mtfie~1|%Lz(1rL;=e7qdGuH1gGtfSB8<6U*)krO+5$A)lc< zh;(2&@FZC2%d{ThCB!E5F~124L=Owk-R0nXLk6;&=r2oF@7E2~&C%#p28Befmxn37 zs4_HaLxy3S?l1LOWu|N#sl;EAH>8oW-{o>ms(yvGkz%V%E2rrp#9aI`*i|WkMLyO2 z&@tcIq%5aMS9l$GD?Q3?o?%TAV&KTy--5rQ-&ZGoE&AJ4!G~Z;G7CLVNN^1U-}sZ= zLl_rUD*Z@$AvfS@5XP=`8QA^MG&GS~rx>dqr9PxSqc$mO$cHJyG!6A94ZO+hGtI}_ z$g0lB_e&GVzIYEwOW6>`GsP)Yj=E5pExS*5qJAUFu=?1aNMC3R|Cxz*?6nq_*^8;d zVfpFVpVGUhoJzoAn?xBSD`sSA;{ZQ1%TxMpUwg;IZ@@>ONWz z*^4fME(72Eb0|*Ep|^ll|CO3$>T@ck;+w34+^&4CJ8pbpV!fvMwDIbzd!~v|4p(fG z-IbN8wrKp-`<1->nXFK9fm}geCN5zs(1*w}=qa#SKLb#Z$#2?Lt$T`HR`xyv-}lyw9*l+gpQa?`hs>R9b`fq578c zxU9bHB)x^6EjdBx(7hri6fq4PT`L}zw<%|fLoMA3p5+y0?@gbcJTd;yn3GY1B2RvM z9lbp(q;z}5`0~lFEzCnZQnARPg0#q7SOIoNHcD#~8Av0PL!Y60$#v2}@*+8;T&S8U z_m{7fk5CWL9MpC(;l87M3(cKOS?11$_h2tCSKD1bPTNY`PkUUoSg}MF33_Z9@f%Se ztBYg_Z5X3#vrR19SYj?z6s8uu$ScYnoZTjKd8%)6TH@K*XFt578YR@v4lh1hcFFp~ z8R585(X;|`uHi28b&(MKBl!n0i+F>a6K}(~M6YP33{gGN#_L~bImPda1?umbk=pmh zsWo(d?|o)^Z86LA$(n__Al(b?6YV+e3vCbWb(L1$n~o<=fNz#(NDhn$msruIu?eNc zMN>=G1D~wTd1DI>X;Bva`WW@`|1okGh zl$?pr#9!mh$h+8kvH(j@=9-flSp8UW zL3)DTNw1ffh&tpGOo_aOWL!(|WgY46>89-stcS`aB@;@f6xFr}xnHv{<*v^en6)#r zcjl(Fiu5^|dD)Q#m+}vRIpFe=UrUc#4a{WM1y&2zFjt9vp=5X|G8W$?nIi2XTdx|a zh*a;@aayDPhY>QZGg=HedY$Q^!L2``Z)3Ql9j|$yZm-G${tz4FV^p`KUnH?~3t|%S z81@EF59gC~j=8$oDNBS7NnHEv{p|2mHF$l`3`z}cK{AiihSLL$FWI4^8Pl7xP!H>to_Y$dUgyn}ToP4p4!A~9LskN!$| zODkmK=_2_AX=~XmIz>4dZzf$Kc}DLhYNCB~JB9sdn$+fMDY+)*Fs+mu9mk<4Tu(0%UqRtRZYvBf=%p9*FFSx`}@Kq=TD@Ue*UB$JM zW5N{j9lQ})Ky?@DOSXYCai7WNh?F>o?}9057`&ewsO}8*M=p>J;kt}Xz7A12FY85D zHam$lu@7K7+5zm??t^!LGg}jwELhEt;coE5u+z@5@ETSJu`XZRUg`X@=Wr36!=&0N zX#+Du;|5{+{jnE*|5^#MCGTg8PE$XQ3a1Z(8TmhR_HBHi%L&hfGfu94_-@Rlh6 zGl%xrW2Pm3+!;WY@RPBQh>Yuw+X_Oo~c8RuF|%G~3aHOk9n&xmEvdj1U@XDgs|?0z!A zxmBDjS?t)06~S%L=GMLt0cMvU+qW(7D2I*>&nJ9)XxhRfitOm|`!zlfQQ%Xkg+0PGfxL%}&< z@sYC}_;2s$6mUnh6Lear3)|t{;JnpBs1KcOn<>9tv5$g*j9XF9io*jc!`_kIqDd}p zEA5EBEMv4`b}KTUjlx3MfoMA($Aazi2#H_^=NesL=d~xY5BLN}LgRTGJQx~;T;)f( z`%2PTnt3UQk81=LO3 z1*Ntk3HnMtW0p~CN;SIiHaEXY`k`zwmJQB7Pj+^cy|S+cyBqJt?#vCoj_hy=FS$k5 zF5w99jbZCa*!gS1L$o0$0I!Lz&N<{sbdzlewO{DTM1oVtEI!^fNfG8mvA>~9&|iWE zE8r6OyHst*dgvE5Y(K;eCMk4`wFND5OPLT^2J^}pL%C3W0iqd7hFRZ(9omj!D+fYe zhUd9o5KC;Ol0`65(U^XSes%5te<(WZ=qS!EjE~Q(OOlNzKoTHG0wFlT-Cc?n+9Jhi zfnvqoJ;91QEncL!yCx8K&-(88%y+-T;c!m=AhSE~yz}0BpXc`wY1l<>4G}4RCI6v% zI@73o)Byi8@HmUekIG7P8kNdMQ8%#}QhlPgoCqi1J?2GaH!XYQW&Y!aJ6tGp4P3Mr z(3j#BbRCmme;#ndTP5AaSHisdXLP7~5J9H`bfT4UYc|cZn(w&6F`EPMYq`#b_n5k*Xrs{g;e;d`pR;bT{z`d{-@T z3plh}v+u1LUc!IDdP=zHX$S6+xh^XsfMO#+(~Vsv_9gQ$j9q{oCwt4ag$JrvW8_ZY zZG{NzCRXNJ9~9^ArML~SQ=>hVjl_3#9MVxq#G`~R2&Mh(eXPHzTD`31S0n*DEiV)Q z(WdYxxXp&muJ6P|GG84p^p|L6Ah-)JDAoR+mLdLD+;|;_l-PO8EcOew8(-)gr&-V3 zW(DZBcT+!dJ1IAE!FNMn$D2=w;HMlD^*t$(UyDv9&xvCt4{cInnJVrU|5E9roy%*H ziJD{Hoy1SxRN|HXYjujnB&_!}2Iuz|bu4pS9>zb@40o&KMP!4Ls*83H7rxL=@PtlN zwH8m+TmPKESNsO`J(Y|G`j^m8*-V9}NO)4eVY7v$ax+hq;h5$J-x~1;bPqa+A7u!W zE7@(@QJ$%WzeOB%BlYQCYJxCL%^(s9yU>kUCsz9-=`gvN+37~`ZD<9OjJ}X4c_f`6 zpTXxyow(DKSDvdRYdX7A&^Zht#6l&yUh2zC_YY%s3YW1R&YQ-GB>aojjYv~9TD2%7eN&#o4bpy62gv)RKw^}|=Y9ZBL=2Vg zKf#Rk?_w5HQLb;GfBCLnuWD!hQ{TsN7ilFwqh9dG$dSr<{wj8f_vnH+Nqm7;;wg$* zoh{S@59)Mq5ga3Za(%WnS;K6W=6m-Ut}0{2HpFl|U;V#Bx0^bTmY`zGp#M+`gsXU^ z*axHaKfB*jr;sAPJ)Eg7$qn#0p%0!Tu0-Ga4x`EOcHxxrQj5aX_~r|F5}1Ijp#&n znHpr=%;|(fIt}Pz&IxO=Qgu7>0H4g)#&6<3yDi|_Jm4)xa{1r!Xk@hau9C}dg}rSK zr$zFVbL&`Ok3JA>&@yg*v%W#lOKRr3t?>Nw2`S6{AzTp%n)Bc0{! zLOI6&v-_2Afcro0OI1gIxU*^XUoOP)FL%6Rbxpq5+I`y-C@c^Jp;+uFOww&3Pb0;| z5_*AYo;J#m6_94#X&M#SJM?hSx!~Ht=R;Gh^(^y(!vhzYUm1e+1-fz6GxRyt1-gR^ zR1+U1bo37uUH(&^j-FHHe&-$k)v_s-Z^|2&45&1hZ!gNPI_1o>d&?W;UUB`+&0^2H z4mxkTSBc%EmufoRoEnKQBvivV<7rI`%hqUTP(7<;T^@ESreAp5(DNbRgS&=Gwyl9b z2W_!t8+w?3)yFbVnJ~>=stqzh`icr_eet^Rv+&;au?BbEardn$scBKrD$i9GRuEr; zIA~A8_ms?S6(ef?u;*18tMVMj9KSf5dFEj!ka}_%v5`o^{?>=U=e!%(%XBvMS=fuv z&x4A) zXQ;#v(EE+=gCrXZC*%jV?a^;)rAN04`6p<6RAp4puwfy?1C|EO5A0?#8&dTPv^L{> z%^`dwRzQxYFOVMWk^D;jQ}QUuLK&a#7o2~(`c&;Ie^))Av~Ko)Ic!b7D>AS7XI)lM zMLXxP;&oYFOIlQ~u4!U_=0EKF>TN}2QrpNx5;N@y39;?6UJPs(J)jN}VGB=)ZWcZ! z^h|hK((098YCV=+e=-;0gN`fn8fNb07R0y-Qs& zj;YQ+YWfzx&t08+v3iL+ugL#x-REs3N%j$CBMWBb zrYpAL!JDi{tgV9$bw0(gwSNeE9{4S^aV%52BFJOiVwo6}WHlM1^;Md1SOEw)Nnhd( znJ#1!+FS`n1YA<%6esu5H`K5AlYH%*w0>IcX@iP@4c!4M}>vm4rvqdJ;DLmf22eN1 zE9hO=D_z(f6hYEugP6_NT$ZJN-w}WTYdR6=9$h#r^Y&Sx_hkpn^ zXj=dkca`Zm^pxzHTbd@C!(?j=M^}O;{UhZjUZHK}u~INv8(V{10H1taE}DDH{^7p| z9?lDuS(QIm#*{TIPjtMspDmu1pH-w*-L!A6=uu8Wty|CK^iJdtaw~=VNP9e$2%zQ~ zmj!gP3^lE^o(ov&Obu5Xerp1*{i!TYF2=_NkYbkf*^31s|+h33YAdDe8}kETVY zodIJ_3rydPT@4#_TbV}mLCqDW7qg$7Pa3H1*nDy-(^_*8?@fYU3`@n^69KNd<%4rk+9VazEse7?WB&&p|G~Q7x0yf8OuA$VEo5@f1g>t+7 zq29(m&a2`6as>#Joe$g-yo>!BZ+G7d?+G@CUFokB{^U*Kd-<;NQh5h;RXapOtS7Rt z$9Na=Pm(3aP|cVoWG1-bH&Cr<6BS6;Vrs*63?Z}0ASQx(%CyzYCYLZbH5he{x}!Zu z&7<1UiDU?U8H=Vj;R`T3)(J>XCSdSfRUGON0a5x0@qDz@jcdWp<@MYt?@m_sM)-I? z;veh(&l}HCz9Igzyp>JkukpJ@H@{E%Lw+IcRp+VmkvZxBs7r@H-6-IViLSUEyf#yb z7GxS|&R@YKJB=)+hf)gl3tdT_plgw-%rEpldWfcyZcnpRFcU?N0)C2wzXykK3{DeC z=u#jG4OifQ1X@d~G(|lj-4H{icj9!B7di`Go)U%&mFx)qy|4rv8gcwvp{xIsu#sQM z+lAf2Ga*k1mM2P!Bua59cjTt(NuWAaAhVD^(Y+Xg@fe0_p!e6FD8&cj9|(jPOHC^(UdB9YiZ~DG@MtUTXiPk#)?o+mFZeL@GLeQ31FldGrcqz3Pm$kM z0%lOZt5+mRIU%Jh2c^#9ZsnxVP?{}25>^VW<(E5dfI`*ybmwZbIkWYyXlxR?J zbQ60aMo`;LL?}?5%u^>TTR^wj2FPERuvG|xY0(n&8=N3r*daU+Ps8ftpP}wwM|2@t zp#Kv2p!U`h3YG|pvQ*+FR4n_5H~1T532wt?;N#%gO+gPJW6%}oA*DTjSqcCoUMC<_ zbyQZVon(WuTU{a{>OuK2=uu+D$?^%+Ba+fqjJ;e4qA2jGBiH#Sf@Cz$W7kA<7YTj&w{-RUe4mfbR7LxJM??f{u}2Aq#}Q$Ypha0{S;KU2y{MXQ{kO zy{rUc9$^FiT79W>Ko?1!2m-YHG59?A24eBP$b9)Nd0cjY@}{1$heVXCSRkC%qu_gA zi)N|4|L;!*n$;O>F0h1p;cbvBz-ozr>us(a!)uH4$>q?uxkyAxZIC~a+RAc#lgOYy zC_Ru(z)U$SJw^s2qotMTFyys33m2qR(4`)gwWv>dPT?3pnGr( zHADTzcP0Ommf=mMqgW{`;2HQ%zmu#bi|}1($y-XXG#$rOyIhYN!dXG*_gESPF6D+q zn*X{Mbe{ZUQxAtqUZpaS55gP7t~u}BMvX##K{jCnMN+fdKS?`_yCGLGuXr;uMjis6 ze3KBZH?wn519YR)lys#Qnk2lX-U3}|4&p`*%K6f1%mu4{yVw?VH%<8E_>U3(Tzlm{!_*{Ic z^@ys}+kBBdhx@KJE0IfE|}=VDLLR_JI(@U#V$ z_68tBwNqQ6o#ZQoi9LcJ5|*Oi8{r=!`#^!8rq#=Rge7zrkya-#AH)Wrd2fmBBNU}X zaDr0rsJazzqHN}ulGCxCz-5Z#XOQ!R2Hpz8TFFpV_d597!cR(IW;|QNJpDP0`?RFxN8>Kb^LZ6IGLC$k; z(L+MCdYc$1IpGX42|EN&xCXqbRq{PNir+{Klbrl3atWv&mm)2(Z`f&2VY{U;au+&K zy#e~g`*I49u~y+n)!O(xrkej1y$$cIyUc2N5BdYP08{7*XexafH1Ago!__A;i;W=e zVDEvzHAMK6`ANF281UZ^6CbEd)^y_ba4Ep4yCZ!PbA$wOxtIWI(M;b1PlOOFw}tn8 zG`rT%3R(Uqz6yD~kij07ddmsIGdZ6atd0S4(lDYOvyXg6(G0C`25!bUa0Z(73EBYb zQq3E~5CdXttubf^82d7QczgDvR}eMeIq9eVL^PEi%IRuds60*l8)<}hiRX^Az!wXh z`mc`d-caY#>X^zC)my7tRG+V2Y=7V$>pbMx>^uzwoF3{I@f_5wk!lo`K$VfawnWQV z?ijXMMg}|%B*J{Qxk2f{#ipaS;E;)dzZ(+`X~tl~cfC&gke)+-qSIjQyi`A7py>yn zLzJ=#d4vwfDrAeiRYJtg(nIdBFWvpr`@prJW^>t?(!CXb6~z=DEm)EDI=6epXzzQw zH={F*d^qv8_=9V^Ck;-ht%%*!bDhO7G@wcNwV*iTg5bjX-neIBOM~Q~zrzEp#Y32m<@9Q%Si9#Gjr(h^T z`54!DPpB)Uf-Zhi;LZB@<@);upQfj`&*Q3dOO$uFUR^I5sOHM&YksWiDqJDrsGbI0 zz+77k@U$JMg@uic?wUBS+2Oc~pzx6PwGyJ11=}r$_0UbC_ksRrzYquRlkReF?+Vud zU$h!a>_VPOozQ6Ae}cS_o)K2^hDpHTwRgMSaM z6S_AlJ^EN=Zm2o5Nth5|)jz~-!aCOC8|*-f^FAGU9sgqa)9(LWy3+Kq|4^bza1KPWM_2T`MH_p4jfcxBSLr0>bwlCl$;H*4GaPScEt8q=?ViNPE7 z`Qj~CKi4UDPy5&6n*|F>{Z*6fN2@}rf?W;eF=T7{m0lm17rwUk>N?M3XVpKFU~Zfp zmlquzF*DpB)Kzy+Y47jozFN5^`~64F>lu%g+m9~xIs4c7YySk@>iIG`C+pL+`}Su~ zJ(XeRpd_+Fe=M$9qOPITjP9D+EwlBgWJ}V$#7~KJlZQ6i7$l;Z@?E@%yx88oWMAIo zoJpAtGCpMX%ez)Qx~!z)lY{m3AS~9<2uH-w@F5NAww&DhDts!X#jNIr#0T|C>iEKV z>qzYoq_xjhwY4bo`||hA9vkl@++hDnJ)QDbpJQ(?oVXTouOcI`GNMrT&YJ&M(8J_L z2_qt3#gA$8u?>@w(e-ri1s%?`%xpfh;l{}D@PN32s1``wvi@bG9Cync8FfAn{I>S% zlJ^Tf_RLN#{!(_%F@Zaej@1tfG1c3YP?Ypj^3_%sQ@*C`Zr`=-$rd*g>emmcGbZ#` z^F(5+xXoQyz97f?enN%C*(R^j8u-w&}c<0<_se;&QG zwXx&)M(64-YqhW=+p@IP^v>~}M>e|^R~lh~N<7n=XZ;2FuO#X_nK!=ZLEhBw7rqb5 zT=<##RFmbcrn#nKF;Yss(brdE>X5r99=v`?y?yAg8yM=h@<){w zA+8QxQtHN!Y4A09Ve*)S%oe-aKW+9kHawzz@McCu-)JzC0iRa(?Ax}?mZh=fJqxzv zm~s~6?#ydi-pLg!HX~MR-x_Vk$Y4v{(&n)#sVyHiotF4xlA{UPXmb42y33*iEJLVf z^g>2aTL94LTEV+)>ie>ctk1pPB|RJY)c&OXWA4Gy{~Fz|_W*m=`PGQ81Kh)PPYjpb zy=y+|mBayU3K}ksYgd0jLQKLR@r~=Zh)3h2qmLUAdD9ILur^i$b7`$z{@ zzM$x9@yn{6j^~bF90xpGk*20gwz~lrOshiLH$2%w*X(m6d*l9%g?im$mc;I^OGobx zIK<3=-KH+STxeV%B=H8C?|+Q$^s(>2^sFEu78Ze^W^L1~%_jnQ(&EHHKCo9=~< zAOL&MsXARf#r?16vLm5tVfA_MSj9#?&{mqp2OP4Bw!E0IguRJ%8!xN>sm`NFXCzuj zS4WKiI)r8!wo_RnPUWBYh3c0@Kjt3Hp7rhcr>xH*>F+)-d$;0s?wjB@r=EpBx&OT7 z#|9ZOrICn8X8KoDXoW*oC45ozi(2MbBxZHw@z9{i2KAhEn?#3(zJaN5XVVsw5mYPn zs0w)N>t0J z=fRnvd^w?81{H0Qa*t@Y6P?(G_t6%;{$&$Q~97}FS-s;Tz8S}GlXJ>vN zm{In9O_n`3x%?M)vhdyez>(uAk*m?8G-Zqq7-e~C$~OE9yKGLd9MLrVNl=>Qxv{!Lh?O!Y$@Pt&pE2;zT{=Vh=OY)2R-SIzB=4w64HrXo1xmL;##4cQ%?gUfe z{K2ln=i##)+a;fNGS-Uw>~(q{N+0n0h)#HGkHe0ttKCJ|5N;wr#uY0kTkGPzy{%ja zy}j@*76U$s{}HcE5o#Y`XHO)DksqlQOaQeK?*Yu|x!P-n(@YO!DD@7X2~OMTrYvfJ zFoD3CHR?aYW$+daqBs@9W-AT3Pt-VSFU+ft`#ON%FoEvNcR<3ZrXDvDscRw}SGx0; zRfKMZP7@^MkoZuVq{v)tT{v>pqxT)578-7_Cw&5}G!*U2q-Z;M$2v;Yqw0CXZg8~z zjo0&EWgDmuU>bH(vHABwzoE4{n;(SkA@ijQUt=VQijW1af;)qqp#S4X^2-%f6GAs} zonxEPHG~hHA^t&bMDvv8f*JYMQlyOHIDEDCo}$BcfwuZC(^@?Wq@MPe-Jj3hA-$G{ zh>x9%6)3lS;N;caQ|E$$!_7IQRm3s0w|o|Tt(;R2VUg-6@r*D4qlsONhMg}@ftl}V zaBRooG1OcCO7sU*hkf#o^vnnxqUj-C^qxk>62BnV*&=Er9nKyUF6(7!hx3bo6KgfC z`K~eo>Ixif<%kh4>tCVq@-=C;a*8{o{sMUrrD7>|kv1Y1{G@jeaf7*lRjLhn0uL9L z3wMc0_%S*izwAHHpVdz!gQVWlN$O9ipQaKsG_jy+ZCmrRay8&H(Vd!6bC+7r{Om%R zXwazNBD1~A{CPGTw!^CtTM-w~bzGbvk{fhco<%IJZX~m$z1RSZ=b?ilwBdRagNeD+ zAvhz(5?aiRa5U&rZb~B5idgHjpgv;34*TMG$yB*-;e|630Zv=j! zrm)$*RI$BYqPmEQ>M?pe7US2j9}P{B?a&jz)Y0aq-Xh^8ww24)kEiyLuRURSKdQp} zP&AmL^|}0LE?@11oS{ppB42Gh9{CqI@1v#R)GesG5z{08QsFQfhIPg+h@{$%_><@- z5W)bMEpDQBxZe;wIgGQDtZeg-kh;jlrW`F&^`BIr`-$5tYz37`V!w-8+oSsVBe%)>dDoM+Q9MBm9%_56sWP zEQwHW^7nK*5s_Efj@mk8JEW^zASd`ZG1>Znx+3odCJ0aNL_F*PsF95FSMU*z)pwHS zcvdiz$)@Th{ucHV{lK?folG+13wf!pA>jw+{$kBzZiRdhn}Ilpqw+>#D*BzxM+XzJ z_!2Q)*+22LK5_R zeCS%rjBFO?Dhpw^AAz>gJXe0GJV9H{F5#i$N9?fKhbAj^3A1-2lcc%nKZfMcJUY#j z1-+0D97c6ihvS>M{@@r)f$DjY_pbkE>OV#&lnN7(^NtdYQ$JcYc_Nf0f%*L3>|Nmj zHo)Km6-NlxoE^))6BnAV!Av+AyTNo6+rd*lkl4wZnL;iUGU7lj=( z!E{&gx%;?DG_N(ubzgQHQrDTA-3h$$I;r~QVBi(3?I0iVqXZ+8cmCAOeNCTQ3 zK1UH$3;$Z)tKEywl-G#@ULz(FHqfEo2jKI&hOxS8+7J82PN6TL`2 zsDbK&w(2+ep_r!*05bm{@TMP8{sC_9j!v@ zp)#2%|02c+KcI8)kCGGgV3EiQxffec1|kT#op=bdtQlMqwij=UH35azI>gS00>8Z% zXyX2Xj_)`*82ZCE<+@0JwJkOZIf5x_qF95_5=m@P1gRAP$}Dt%vQEiS_91&9Y2qe& zRJ;bn{seI%u+R^q7trZ=Q%F4MF7H%kK$1iiJ`_t-*J2VjmDni#BfIcZ%pX9NFyP_% z88rYqjV*+Z`BCZ-I6-?sX2Aw^z7z?G2UW;y?4H_O+@W-m76Ku97W6f9<;!9|>@~k4 zC(&7;L+dY-+!4?lL9z{=t}X+A**LDY`j>oA?jXN`=OR!&F8nQ?6Q`o*)HmuKWvy}w zpNrk3W}^-8WmE~c1@xc-{DNulh6GNwV8$}3^hTl&H4rq|!^jRqDRTxKk3Q0Z-U1!& zD`GpHhzpnqT3UwOhSr882t)}&HTX6l>4c(YajzfeuL_sh%ihn>dHJh4rKYXxpsTm@ zjD4a@=N?d#U7h0cxpp`wJKs5G)~u`CT$SaR0o`ngPS7ql?J(|PPEr{iBgLNHKzo#ZgJYeeqB^|#x#P5JmFK1_%C){Cyttsip4~X-c#g54 zSyuP2|9yFrx5~SXEY(9nZter@%FLk00Z$AMjLDYG=AFhN+N+@7n8PJY6Tyc(jJQg5 zG>x*98188OrrK~CeQbISM7Mb%i*1)I!G_cNCdP(_cDkl?eR>`}MVqX-j-@LHkgh-) z%~8ighCn}Tp}5&Q+da=+=G<>Dah`JzagVJoE*zXE=PxdNlzTVJo7FOF{dZ5shxGTK zhJ77bIYxV>R%~s5t)+33<08ZL=6LM@%~*XPu)p%CKI#tNAFlI`v5upz8#1nM9&$Hg zOxPvc7V95|olJsmvXM0Rhk1L0fL3N3@LQHa-!GQfqMdBK1Y9StW+F2ZsJN5R`GgsK zCchCwsqIVxs06F|$^MT1iM|@=V8?gIA$xGe^`f`A7qdouuTF27UiEENHklpwea+`h zAAiiT2w}Ebbwe9RBt<57Ojs2e%S@2^32*r++(%yK1GyQ#>F(pMCq9?R;Ug>=5zC`p z5gS8~2HZ2g(9D69iL3Z#=DEI+X`;bOr=e}6aXg_I$zIxMb7^o|SaR5*z(oB7;s|C! z|A4!GnZf8H@(mG>_LN6+tGqotQNE)d)cLgPRQdXn%)l0^4=Y-F41rry% ztk#>_=OeXYM}udWzhE1E16-?s3Gj{V4tsS=VA%F2+NnRP+d)%{=owpdbV*c9cpuwL zqf8uAE9BuyZ*&P!PG8sNYV_nVq>Eh1C$hhIPrG8BmG*VjJQl+U^>6VbDE1$rXq*%-MS8zX{IR*gKDH&=mK4W z>5ieJH7HDsoEer4zP&x@9c&EN1(a}4>G_6RMvL}6HbKUO9<0Oj6`W4<>=ACS>%W?Z zWw%P(mo+UPU%Icjf6?B8_?*Mv=4E(3SA8>hG~v;S;fcRBY1y_zo4{H%V%L(titZN| z7ls$Vab!pzi3Rj}hNnu5(93a@V*WC|179z z;Cbz-=WFE~E z?nwKqEr!%yjlHcHpL;(`$TH?RN<-Wl#HTp~o}qViEF)@TOj}JuO}UnXHmB{H+03|M zl422t!>L&Dc)6#@NcuKorwDqn=B@TNGk|;8Aoh&EqCG4^`lot%Xk$3I1V3#e$!lf)3sM@@UiK*iD? zU?fg5{%mmsmxpt~rwtrFL+vkR`}|JKUEuE|`OqY46tkY`N%lpL!8>BBIuEE14Xi7I zvOC*unMN z$FvTvWzbSK<$>?3^Q>!!Z!KggbOBa^ET0i4!Tfs?e?po}b~bLYz6tmN_`F#mw`^Ul z(~OgJiJCf$lUjiND~%ME37gry?hzHKrN##4^~PJ8K6t(u;&1JmTa#X~ zpzL;W_ky)K)c4X)NpFw8%6a|Q`}todmFh`b12koG$EO`9cC)68jgiTno|83f#iFtc zrLU^?I(EBL+_zjuYr?DFyGj+mZhwF$>{HD6hNtSEiE3+&(xjk$#5SCtz0DowTg!>+ zLMh5O(V6J{5?_Gh-w*Q6Frk$+0M!C zf4ybgRy7)W{)xJLJ!U+u4`&YGi4D^U#lG)`pDQqV`S=R;&g5FUA6b9tx(}TWe3~qmqnL(tLynlDFqv$i8N0M z7-9YdBt+cY)|MM`GVEFe4uru-eKErNcRSiuzpin+e)Jy^4kGu7^TZYO|0!1t_L+DC zj*_}`1l*@JY)_|JR$SBvx|Y?2!38~X2WQ5lU-+ba9GlTDf4=jhW=P#t$(@=GN(f2J zj-MW!NFTx)l40~eWDZP;?{i;k0!mutzt8dIt|;H>x8bL?!@*aE8Alr18voVn4Na^c zLxLlAg^mqqt!s(fAdw^9KEEd4@xepzMZjFyEq)O46c-*wB{REC&BOZEw$?fyS{jgM zEMzp$5x9h2R2zWWftM!xdEdu)^q&nDp?xfY3lJ*d0`A)X|+4 z+M8cBrs7QH!0PTbX|B8e1$+$K#cT1M=Jz0Gjn)!kTLvV~QpmOFU>wb)QQhbzObxIi zYanau5?=~_%XXfLo}+vVV!HMoOyVZ!AL@&=mo$`iirxpT)2UXAaUKndBD6oUP1cF$ z<(23VYywP&KPmm-gnUA3CO+mSvZvUdTmoC)v&pfb>Qrf?!jJj>{CTWg+% zZ^%oe(c%Sh9nift(S3Ap3}a0#jr+9{9jbY)l{LRH^O!kI5AasL#&!5^Q2Y77`+G#( z$42;nfb~7p?RILN8*2_#t}hQPTUv6e@K}CS{!ZAb5^~ZC-jwaPcNXX923hOt^5}6E zSI8#Yi2!@ptvXe;9)>myTyA2iN=V>(4ijiS7vLM?U+8&N-L=wNJ3iMoM&oiF^j{ZC_Wg*c6lH(c^i|}5EOD#huql=X$@;7BN z%Hcnd6Ub2FGByPB;5Wz}plu>Ge`s0FFs2e{5^0zlyavGY5?=aS`!hTj>>Vr5R}@uD zEx%r}rEo?5lbnlL1HTRXmYwrovAJrE$Dy9W^F%v8i{2ajIb=(aExfwU?P$byK5$QH zz0jLL8;&-Uy8giGP^E=Z9e%F&xU1Z*R1L2Bt>!mZy!)Q}t$&m72DpMV)Sl!d?Nj|R zgUdMFG{fA|Ix1+8tyf@(b+hFTaPJ!$-smpq4rymG@l+001{8o%(p_N(@8(8vH$Z3o zw=@JMs;}h!!2er_T2L!6jMkFFaS<}$uE_VLK&dqsA10rXJ2LRuqI`l+vBa=>7W*BO` z1W8ZPQWbaJ6XhQ3HF{?`^Q+(6i=kTj>UiYp%uW)#Ql9bz{Yc!SPwNMpj+u5E&sxxs zX5lwN*8@xYr;w+?3xmd3P8)oh{>%VsG(HD;1X)#KLI?f;$NH;%ketoHXkYgba%}!&Rz)RLk+Lu&)MxrR}np`PTLZ-j3?=*Cp zue$$qeswkT9`=>^dTG)CIcHLk43&z=oMf$_$ zHo-4#bFGx+v~{(0sPUtoF|^kGLk>chBFmKwF%7hI&A5S3J+AR)`PX<6&vbtyQ6qNZ zSIRzoHT8n*%`7wi6u<=r2Zh*n2Db?P7*-zkCd3@HC7{~+v!%fBf2Xa9^5b8Din{^& z6#Y4i|EMR=eFXlFa!#y?sJUg2cmC~IXCGIyw7N^pIdDUARdJQ$D)v^SR{c|LuqU}^ z@_ufKqo=*8SfRf3++(TKN_(j+&YSbs7TD(c`H_td0%Q-tM^^|9JsCWY%>QVBvJw~Sh8U1Kt8 zGBsfqb3m~<**YXBGw4Xbf0pScr?I}-Y)LeD2`6#`#mNWQ2k`>?*0;_-lFW3NH7RkeLr^~CCn)h(*;Ra(na%jT5tsn}6|sB}YV^RfkHqsrQp zJq2RV`%&gqwsK4TgFVANA^ryJboM0>f*OhElt0i-cqUn* zVGK4PA#JfXuvG`Qv^5XR4hjl!*%k%Vm>-y&22wvj6GHXCAH&omQR=~U@s4p#b$K0+ zt1Xqu<)IaKDw4|&l*W}$f91y`T^)>Sxc5QXW*n3visc2hSRxzTyUui_i#*&b-LuEgf z?k##+$d*2+SW$hgrk$sS?~~)YW3RA;=z|-jR?K2&%#31l1VpP`++H-f{AY^5A&?aLT9$OS9C3P zey(0nd8e{&m8BxCv|kxoGQDJC$@&sQ>1+5mt)x-eKNWj{n)1cJ&(W%8J%m;+C7%G3 zYd%O@F#OdKi?e z7Uixw4ywraf=;-=_ZQDf?Nv;Pk_ELJJj>I7i=4`g(-!N88GbMjr@mXOr=a zwk~s@M9D~EE|#jEm!=CocDbMM`8<`5Am>C_LoMyb8ls|3N&oWkmCGxCDm!1&wc?-3 z>Iy43@yEMaAMF|H`OW*9cZ0)hi(Dh}*uT0W?FwcHV=#aR{xxU4$sXtU3# zm_A#K=C_(u+6(j8o|<(0Ejk$KfKEY7=w-E+Gzu7`@0A~wX=-zfL~kl2T156>ex#pM zLzqEYOuI#MQTI@PPdiluWIFi6fw!MWuOL3)WX_O+qzFFVH_1JaYs%ht>T51m^Y+V* zPA(ttnw!-8UisYqYjtt?lgdu6=8jZHjAxR6fj`QX?E5Cz!^0t`CS9Mc6OkRzF%+MC)*+70v*>{g*+6JkTcOF^t93*8w#4rWO6eo zZq5Mvq$7NHpXd?fIwlCxf+i9*^gQkFR39MFwAIX|mf^LSzd)Zl7S?=Md^eob#$#4} zd*Vm60NaJPrb1HX|MZ(-^(3MkL&piy-{eU--Hd6mLgT?KK28!5ED5&=(qEPmZ0Vo z(a(|!%+5#30=x&yLoM|b*~PcPBa}+EANElyCQb<><$P7A-iFzX7HGF-ToHB32YQ0f zk9?w9D!&W&Koj~m5h@mogNdhb54#cjg`Ie9e78Rz5-<|f=gJFI1@-$?@jvix?2_Zj zJh2n=0$C+Z)?62z)F@TK_A6KMZt6I#2}|-N8EL5*oEDStfh8ft-PnT`Di?@E=W4R%Q<)Gw#h;EfmOP9!pNR9YR z8|}FayHkDeW*2GnQYZNwHr@X&@MsOFEwyIxjF^Td!_4osT8CQ-KH6PKtn8K|=>pm0 zJ866kbE5%@m43+of~N^Rv3z2#P=ZwopFlSb)gCt9Ut4GLZNi`8Pk`Y3iEb%A;%(X= zg=UBf&WD3acexvVjT_C3#}e2}WR!A20ktw3MLlNILwsc{m8 zO+@m6JbhojjseR_$v|o$%f(hO{R!}2*J1D+MW6{-iPT@ZN|zwN@{f_+=ip;>r19dnNya_m;q3Z;m*+P7=?3XI3d$6OtMQb5Dx#`pg#U><^ap+Oy zpfCYR(2nsB&_wYJf6%bmn~$#7W_bUSlJz(F#@uKfi+%y8x(aEv&FRm`Twx>g)?WzM zc7O?zQ@KpdVJTTcfE%m;arzOznB2&vp$D;>=s0v3-%>5bIx2J2I(S7}Ue`oQKuBo~~F-sTlhhRHD&(G8j;yXzf z$W-wl(O&r=j0F$qS9TkJ-ta*9Kihd7ur<0X8=0QU4}yY40UIlu9Lp`oDJ%eJ@wLeH z@&+LiyhR>Js2oMsAm`L~$V|ngVssOs4Nl{!qFe5Q#xhM@pJ+jvNNDBr=s~npsLLGY z?qGv3Mp-4X$^!a6OkXSN+uZ`ux zp0-2YMrHc0YWpL!N7Xz+*Q52>I+|DLHD41=C#j*7qZOT9=|J#@?GsC+AGIN@L6`08 zOE$+T|5N=yb_ID;S;xMF->cR1^dIMQ15ZM?tFdMcp6ta9mxac@`{u#kjyOurmIsTc znOG!EC?@{l{{rfN0CrtW;a@S^#jDguJ`3rC4^=wiC4M9GSQ-H<`8mE8rq{dlRh$n} zaF=om>1-ek4B@USGnny8vhN;{-cmg$Kp%Fdsv7HJz696YO#GlU!&{?AuoAwzQFVM5 z-&(?4gt7^;m|K%3cNuz*jsu6OXT#9{uK+*vzVY#_!_bA`IdP3#1uyT{2rIUT)0y(2%s z&B!hw+K0h3*$X+@pNYk!PV5D{+9sf%tdxPNj1xX^_v350AxJeAjWx%HardQmbVKx; zkc1k*DfbgnuJ{yHF2!BgUbRNj;jwa=JQ!Jr&lRG90{=>O^IPOdIu$%Ljg(mQ8tjB^ z;7Vqqqs1?HEV0D%SZz;zmhLDG)OYG{66DtN7D$mv@)LB1x>Q&qZK5H8o`{oAh+mZ6 z@Jgl)=da1f0|$EyYWTe1f}g_{fgvg6ng<-3{)vw~hoTjdBitm@<6cp1Hj@IW+v zEv{fHm0|ofxfzn64wed`-kq+F^_Kw4cnL5qud4;vPtqA73_k&HybV+*LJ)qF4{-0P z@pK7z0sp_pE|g!Zb_^n=z}HnJPm#Jvhp2_*G%R1@#JdCnbo*Vz2lYMpL9QVE*TG#FF894h#vvVKNa+Rt-e~C# zRz|D=w^%&Tu``7*`6K!k+>5igmtuxkA1Fq(1)2ZiAH_vV^ZfPP8~O2moU0aofb`DV z;?GJu*ky(=x5zJ$FO`S2rdR1Tx*s%QI+gr`^3x{>M)R{~5*f}spdXQS7=#+Z>|vh3 z*|jhE3*85UF+IM6l!=4^Gn>OgB^^`fv(j!nb zR#WXj!H@*A%3Z`>T&wA>IY=$0Koh4e1Xb-#ZEI~y<94&vJRfL!)c;tz3iv3luRA+4 z>%Q5I3xq&$cPJEhch^Fp1&T{?cM23j@!+n-rMOE%j2OgaH!E3}k?(~6_nY6^?8tj> z=Dm6M-gD1Uy;gr#DHJk=L{&@PK^`gkKrMr3cL%>Zu#8FZHggYjXzdf=duv=-VNR$x zQ!%JyO2ONL_TQ=R53>(svdOJKbkE%bJ@F@CB^p3(B$FjObTgv!s!fQ^0c&xP;6I|j zBt}*tata8(9lH$v*N$wCfDjJ=^HeZ$5Qj;}Y2K>dDMB=fsz{kZmZw;u8mN&6)zM|j zze$G6c|~U^=$w$Om(7#C5q6fjIc^mOMqd)o6YV@#E$ z^NQ_xjdHeUd`U0=nwK&1Q|lMeACpQG#iw<3v<>tt3{UkF4NIazW6##CsF9$}_ea^h z4v%Y#<0?MS`5$hz#G^_Y!}!8E7)D`6qm$b zLeEq$)5jX}gZqWZ^}DpE)CTPVMZVL^?9*KdIDlh@|` z)1|bohvL-Pyx#dsOTLx8EqzivtLR#OLiYWPiZA6KKY#13B|4t&bhmbgkXoAYDn?#T zS8zSS;x*fym&1QpmU_x06BJ4l6b@a-a9`Lp-9cGfh0HJ@%&o~H6FmP`ZmjI)pdAfe zIm}gYi1M4Xjo>zMOnyrLJ!Dbn)3EH2>Or$*=jruikZ>bFKXHmn|NB|D-nRI;g)&4~W7APXb#4cgUgg zLhW7s3Bw12K$|b?Exb+V2?RtUpGgd11kU@mi_UBQ!GSXP%H!E1evS8rE3tBC#s1Q- z1;qC+Sr@=C_&L8$PDoPyl)3zyHf2LTbZuYndyFQiT%a?J&2L}Wu2SmpTaN z885r885#a>&CL3SS`~)cbg(DSHrcwga;med%jE3i=@EcZh5s{Wll-o$8}h+$T&5J!v`yr&XHt>6u7CQeh#(Ao9nh9S_*?54O%4`LHNUtOEL-GRl+^S<6D&bB!^S~}oMb7$O7 zVIpq!x7g2rWjrt2V;^qMH7z%ev8D#5iAU;&MXsqcxB1B?BcfQMprT1p)8gsyNo_U% z?9Slrz-jLy|CF}Y-U$v3acVwEULcz{87#ehT_v7Hf_P~M1*^={o(UQp^jFX+hHjz=(!SI9WOKx`pt{hE^o^kJFmXeXmBSAKWkX|D+sP1V`tJX*+!TO!& z|KOBVws5F`oPDJIJGdltS9p!^2>l2767fTDtv-ZGLTZ5bwe>Fbb?5ezJ;c|P_jF5j zGG(@CGvNRd&{nzLII?6>na}*ze$!FUX#ysXvxb(GWLIY0{C=$9pKM>+uJV0hE4oeV z*|N!u=&W$PI>|XEL-)RRil^j=dzp8cW3%fB)gdT2JUQ%g)RLO=_^j{=qUpAi-zKDQ z%nB+oS^w~{!bAGMBNl|-*EEq&lGT;ROaCEugVnbeGc|D3)e3x<@FFSW^wUBHMtGt= zMJx##Bs(t5K!!OJIO?nA&G1reir@wLMmR&!OS4yb77VbRMNQdXTwm=^9G@JQ?31i6 z^H$qv*Bnp0i?=PSh&Ps%_zPy_#$`A8dhSclub2Gy>xFi|-(^?JrS)coPG)+1yYP|t z)H!#BZ3p`n=+N5yJkd2UdHfSp5q`49+t@3N4FCA>Mx`j_{|~yNdFmr zCr0_^!m&cLSg&}k+M%2*osS%x-M(A)8sF8s!A@2R9ioE{cmlouf6xy!|?*w;^I2fF-u9TO^-$-rbg+Nqb0rH1# z`Nz7~I(~EYWIKps)cf?cLT%xjA~u93Yo?1%5b1m|q34nU0}*o|Nf)IRJ`>rby_F+0 zAC;73J96N+1$5q84$e|!?cyr-AMyLWah`u&EA3w`jZ8hupOp_Nmz21^O~{&?p-<_Z z**UPf?)5GQyZqCRZ9X*iXUdk}?9+<3&%P8C)o{<`rXoAFk8CQ`d>5)xLR;26T+~35Iu?*Rt#I&U|Dum;}_MzJHbLCXmCGfmdXx z>{(E+$Wt*rqn3r1YJQP^CfoDFxjs<0SOMPZ&g3MrC&|-ifHrBaSfPwnHj&R3#S!-d z5}+CXL6kGzHOu*1<#^jwTOC`Xb${gpXRssDG^+5>w|zN-G94+lqyhQYWIbBn>!a#+ zuGQkYI}FioMW*px*H4L=XG#&BWoi+7=zj7Unw`*u{X2MATyX6j;h*SU_OB(I^F-e+ z$crn-u#>t21a&&3R^{~?3+J{cL`w1uQdnb6Un6JNKh}N2viJIFv7tyDLR{;$sDI!mH@Ym+2C!BH<~d2IAqesL9f^L--&bD1$f<^4)+ zk$q8})mjXT!j1)(D3($2h!94BXTK#;gKQ&gA?_&gOGZf}rL*M@)i;e@8>bzrxFXDe zFZ8jen)d~=n=+}cqUX3<_X3-eBpJ1Fx8KZmGFRrt>*4-LMB8SW6f!@w}t{bjyj%@pZ%3G$G z!lHt$6`Q?2!9&G{h=};jwFZSxQE8=SdLf6{c%UXXj^D~UeE}z9J!hd@iF|uWJ5`$? zWALQl|FrjHq}U;vAxV%-1cS~J=s3!$?%-IdBo~XP$bXgpD?K3HChIHRL?;uIVN2Z> zj0B&7o}56opgzGq6p9;27K#g~cdW^K#}#cKX_1+uEd|!SwqGn+#$%<%l6xg9OU4#& zFCJMksQAxdz(Z7RiOE-y@QD2U-B!`<&ve+7}-xUrL0uiUi=FfHJig?6$%ol1o3kDIJryQ z6R4?$vU}2F;*HWl$iA#lb(h@~QNmRCCx-}^i!;S(^h`1vT);EId(o5q8w!2{iLHc- zJI|2*1Wzyb6IXB7D(6GTPherbQ)#K3XB%(cZ;Cd~D651%@2j#H(*a{x>9~@+6{irh zXe1U$?Gj2_OR@os%zYItl~5#*mPnFefB!0~P2Iv1qF7js9weB?+rfD9*8d98-}B5O za1bqEuYiSf7Fet!=xrj4v?US3o^8p@1LSLP zf!XXO-)~;4H^$9+hxvFg_1^VLy#?MW{wIOn=+!frO@Xg|A#i@Z;0p`lws3>EeL%r7 zfn|aIft9cmjxup8`Q%-&Q-6fURTJ?R@ni8+@mld! z@qO_zahYg~sIw?i^jR1w1Va&7ohqSv(CerPR3f#U?l0^lTtvsw`{;NfFQi3{g(})l zo&&38ZTcJ5zNfZP-Ka)XG5H=5(Ak2Kh(z6o!bu$8f{SM;a?HW*jG-pMue%A>R+2UC+VR70yfXyFcFue!aGc@YUdZAy(EK8BWKE z-v~25m2bw+0UCdMMKnVj}LiQ>X#%aeQBf zn)D?`p;x!Z7A+B>i9^Ibl`ny|!Z4yg>S4ukp5jQ8!G3&|*p4lyV(+;)^Hl8Dg;1bg zR^G)^P@GQXGm&Fl3%#fV&RC5IfexVx$E}SPq0kxv{FdTV3>LduL?s zhu{;5OE?+`z<3CY;U6)Mp(1324(~t#{8%Dz^f(-&4n8CBEe@Ym|3qL5J$i7}9${G5 zfM|_cwMBV4psc;Hbi!v3w0B>$MlYf_K6~S{$B(5OwrY>gbAdUxX(RmZK`rxGP+@XLksR<*q2$71~PPgM!3 z+RcQ|sx!N>4Ttxtes08f2J2jCQ!7S^s#0FSmL*v7a83t$zZkVmMVad1&Z&d^L~!!) zk1?ja$Nxr*Ik)kxj8~wwQnBv^T+RQLl))KH{A(Nrzdk#a^^ap+w+eXk0*LDy%#cC7Z*f1b3jw65d z5&P53zQ8Eyb4e;sZ)>fVW%E*QgW$qrDZ7|wMjLy$ZBM(~IohdXsL@tdFv zq2;C{SK&K2Lqos;7(-O?tXAhlY+dkPgmNdzJbo}=mn>x((6f<+yjJj#Yf5Gk>-bk> z1NJo7;nuT#5Cc6+_JAT>3BMbt&}YyV6$l@2Wqb#&Gq*z!#q=SMBPu?Ye8wFlTLlh- zs}fIHIt^LAlhFbXxT}FoJ_50QC4U>3qN|9}-}CPbG(yC13GP7?GC6s+1Ny~Z;7?nM zGlT|&><+SA*h;M7DRv5bl$j{davS(A^jJZATILT61PSxGX0(QX$@r=F>{b31Pzep# z&4NZy#F{R;6_|q3m9qB)OA!yb&dep!1qtj(us;r`as(;tAO7|fxWxsHpcfh*WVL(McGo zdO$uET8SP&c#R9p^R^90y|=-ToaP$puk@ev%@!R1uKlgQooJG$G5emY=6Wx$@b&O^ zl(lvKB)v=^227<@I<=buD&n5#piiL~<1J=N=aj#Yk}6eNvod7lVz`?wbWZvr*1mSr=$0u!;fpTH+>N zi=djIEbvuuDR2fDk?BCY%3awLawp&Z+`=Cne2go zB3tTPLadd~ao%II$U>G7kMYD1e*zsbp83z$STtPR%3aBw5l;i!Bf-h>bu}Nrvv-V} z%QV(TyYC?rd!|26eA0eE|riXSd*+^qr7(9MRom~!L$JV1L{6z8Zt&kc5-zf zwnLjmDp|{lMDc7FFT)E1ef&|(bn-hj1bO_~L@%lX*8*AYO8ySAvsY4j+D(5Zn_&FD z$1M;H^hUB$aXd8Y!Ua$x221CWfSA}WIY9>`HxVBW5y*V0fy2aH>Z`P-bdT&1tclT_ zF7V!0$6te|gic|pa1=)N{(KkDUC(yU872wnHUqK7cg%Chxwmq%>!R20M3lpSz_rca z&-JU@2?eI(t{m?{e`uf`JCRuz*u)QH#(NG%p`eldzyK| zUD2-P)-tQhQUWWzTSc__w~AIqNqMi5n9_lT(t@S=a|=(Ebuk^VoONm0!#GN)XpH== ze1n{mZBwt-^$J=Rv^#iRh$AF5#1VWhq(+EJzeA_cdNo_rDay%;5=A5VMd|P2KZXC& zqo~eQfBHJz6kN+-$&hpt`;h06N2`P?!AyP$tMzU7_I1WN`&KryHMXRbEiS&7KjQnb z?6{1Y$tRN5C*S_MB0sUXW5uA#lkW3OICq0sNd<|vN_Qw*8@`0!j3HzHs?jZOa9p?Q zDOn`Y@qmCwnx%PI1oGq4>>6x?yuo*?4JT;<~5QN-IMthi!}X$sN0a?JU-#_`K)KdSqr?%Me8(bL0C`cGw4a~j(A#o^?@Y*k}z4;L-SMc+32}*-Q=~eQ_`ROlmbIPVUh0Jzhm@rgUM|o7UA?R{w z!{}Gl$JX@KIaGg5gOhdF)hvRF=7*qkxkR{$uf;6$_*}oc*Fa-h;@b-z>e_)bftO4f z&rt#CBo!GnBXnittZH{-iee^4Jqo*GxTFk7c2E`Ij964zyJAAgu)NaDQz`X7eR?zH zDfxKqq%ABuc$)gVm?9ecr#UBE&8C^A z@8)}zbv?(K`P51%Sj`N&V>leF4EfiP9^}xy(rwf=*X`9U1}6QAij$ob?&QyV)z%KB z+HV)qW0N|(BVRRpUjJFhiBi0`Nd8K#EyiHM3g5-}%2AJH_t zRd{H`tjNic10qVpo`-bQ-&TGS8;~tq$MxLW$$X=tzV&0}22T*XfsRo;)s7F&3|$=F z75Z}R48^)m8iUFquO)vh`v~TRCBjx@Q*MoSN~Ng$Y##r$IQj8=&Fk$iKE23%v+(n^ z^kd)B%6d3=bK^yQmBWL!g#?9d58W4f30apN_3fdXH%1@Om+SZFtLb0s-m7=Yw~2O; zSJ-TCYu8&xoO6ilt>L-Wz*=$%GTTG+^58GQJ3``v zZv`#T#%TJedF57Rf>Nm}R-RXuDn`m1NE?dx(00KKWC|8AZ@@^>o!cjPML!i^kx5iv zkq?rfUt&0D5E!QDdZ`C0ypm*k6A{9^M#kuUtKW30lrNl?7szpC*7<7A=$N@Zr^$D! zXi@n*+bef9CLo}o!#G{lTVv9$*Y(s^SBq3blrH&0*#YThDB?bnHURhOGRZp0GOiNh%GmW+_Dmx<+b?hr!{%h=0 zAYac5>x)IwmeMnlvy!8d+LA5eFHm_pD}u^_=$PoO$O1lmPTWP(NYY=DBY6c9h}P06 z($Uhs(go7l(qYou(kZfj&|H`-jh8Nyd>8THMe2kMjAl$5-*9&yM{V0L=GVq~Wlc+t z7C$JNQC8cy$&_WzK!2~{{@^_kc*XALw+lQZN38%OZ%1Jl;cQ{Lup!Pb7ab7=qa{+s zS0#<4Ifyx|g>j!RogsZC{R+0gHPRu{=2EvL45|WsrB2CtNk=@FUC5glLEj;Zh+UkX z>F;~u);U{N9<%7d^w_Vw4HQ@YG3hIIn4_$xY=7EMIbXXMc!&FEF?x>UIsqMADA+|7 zQ2)}Cq5Qa8bOh~wU2P)<-sVmR>9}iO)zDNE2lVa z5-N2Huwg6K$@j~e!ax2<@)%M6EO@3P$=?W?O9xXq1vqaMYTp5rc6<+QaqGmvB4qUH_o{YcX4Wgl8+VZ0B3f zm>xpwJ=d6M>~6eb>|p9yF$fupU2KGXnRB|^?3w556ZnLFp3B*Y60#mp4PC|0CEuhk zWN+n%6f$MHa*FD`s#w)ty+=)}AFKA`{66KM%8p8v@**^V`pM_XW=Xq93Pq!ZJE>~| z6Bwu41h#vFp(vXGU;jE&Ytv+t6fBENjW3NgOdm{X$VAGtey#L6{&W4~S>&r5XvS=0 zN5Lo1kUCEv6YZ1uq_1R&a=PZp};Z&RXasA;LOr*Xfr(zwjjsp5{go3(M} zEJu-ZzI&E;sDB}tWB%k!&_ifRJB4oX9cePuDjzA1D#xPb>x1P}u9>Upp!u%8uTEFr zQgg_|`dL*+HD0+K?SE4?UHV!)T(q738;S>wxf6j9-+T8|9x9HPh5Ns%I#{XXU>Nt2|dWLV8XdB1AM%!i1N3QFh>0oYu8t|}lpo}7ee#A>L z^z-CR6m66>Rbpf-Q<~)(nf8EovsS46p!ufx0Bm7DwEub)tE7|y#W|Tx(q23d9Dhp% zbNEwB1?9gsMu74wC3^)cl zffaPGa1Yg5VB`K`EWq#NId9q>*6!v$CZDm9X|2&#KDzvD`Om2PEmKOx8OwIt9($DY zjq8i&w$JD{2im}O>LZuYHAH=ZW!x{{r|7TzL$yskO>a~?aEV%V0j(s7IB5pL0u8F;a@S${fj-{oWI&Dtv$_*u{!$NB;)q-`0@tj z2g^O>bxjM<>t90eB-ARoW6myrEJ3c?Mo7>s%TNUOq z<8S4!kh>dSUS9U8Y*G2W^2f-g4wwt9FDf4bRigLQ@n!o}%nP$Hz3gAWI#g`*l|`0cDc_B=-z}S1P8b>Eq6(?SX>Dq+?aXn__l)%|3p{6>EW@V* z%|BB#5NA)6rzs*;A5|K)S-n96^HDQhOY7cfmuipVNxw{!q3#9DM!7OxIY-e?zE-*x zciqoaq#%oX!(8zfc^0_p0s!&^JrdU->-aTvG#cswKx}v};{$ zJxSgR{xM7iyO19*0IEqiRlG(z0WEJ(y;eHXvwq6OERS?m{jQ(T+S=VoFKS>3d@yr^tQ`Ni_h0Ti#V!M-_q5t(*F9wMNrG zldtxo2Xw$4U!r=V>Z#JIo+|4o2P&q@cOx5ir+BMy2l7<*@IEkAEc7e9uUuyw>2P}o zS$bA~ES^k!-GI?P*(VZ{}3GrQTv+L7)pOg|G7pX`&B^T1)Op z$H*@#7N87EaW+ETL)}7MQ$0alU42b8R5e^BQyow?R=!j;Q7pp~$0mI#$rjHSWkZ?o z5rPBlxb4gyKkdu)^l|TW{%w!0G+7fZYt5S~yr$L_FDnG*8s>>+z2&7P%(kym?=U$J zV&rY-cZ0=ZC~qfLkhxT$@E`F*iBdK}eqS+5nTMxNKk%rG#Z@_|nt&3VR}KS%e+u#< zPs!WJ8QEIdVQEjyd|HT2!W(p3Xu3Xvx7Np1WA}hR;kkD|V$TX!f}^XwUgas1y7VW&o?)4hG_@%J|*~@tV1t?>gF)XE-5lcA!B%)n9|=|oGN4wm6OpeIh$L2Wza!_N9MRnNOlrX4AL-xcTi|QwYX%+RMjqPJ z#8cf<*JA;@!yFIqse`QY4_?~$-q$cN9&y&1h^e+Ejzguoj9g63qF)J5i~5L9iQ7y5 zlMIt4U{>2jItw$}C79dpkcu!ThMI(AwOB6B0t&mnsI_n}ZJ`lf1$6d zKRD2ZSp$6Ec~;GZa!=1 zF;XyZtH8k(5qR&H2XujLfiHnuf#JXbObSd5ybRn8DB!W%2OPi*b{BgQS>n~XzqxCO zV0)mE`2?|!{)k7;Ax_|Ws}QqqE?5tQOdWD5`4%c6CrJmX218{WqWaLVp>9zcx(7X# z-bp{C=h7YNFNm&ds7vG)vI7|newRhqTMKT9dSJ0Vfyl+Lh?}G$p3sOdL-yZc@MVtV zo}zsSun(Eo$;h{_#;RB$at>cI1Az+I%xq;$%t4lAn{$g%%5+YQ7)dY0Uc870EFj+D zp8S`1gV$LijVL9);k_R2s-prm8IKawB!kG2WIOV!;0E|Rrwa_o9qug%!e`i&rJbNZA|xBpiVcwY_b;x=UBZnB_$DZmEk{hf zDz?nvzK=odtvRCYoro=n{#555bE|P*4Cm@1<4KEm8Sn@DQ4bA&6D3k1Zqy9Xqgpt3 zYebIY5L@pnxF>iG^y(7yf{AnojCSt@TM@~d&XwXi=z!2@i+tCAn4ZXGU4|UImxPRT z0Xb2Po_;5CL)51>}Y6Vv#id`+M%IHMM^A&dco%x!dPG!a zA%>;F6&lFr2u9IS;*_c!1Jx;Yy6T>?7qS?>3N|7uda>)Ry`lYMWj}j@YmCR@yUwI? zU9opZ`0_ttton^T&TJ1Dn7^PebBu2zm_zNPj|wd!LbOj<52FZr0C5j3aGbdtDEB?^ zk^Z6n1Aa{)Gq3^(q9urCHXvpYV~Bk20I)jSp&j;=$zhLh!=ZOGf&7)WVPyDS1jh?9 z>x-dT)`SS==W{LD+ROvS6xhckF=5CKy9PD1zQlOq1TY;71ii@d;98nQ&crzGA!n0W zWJA)4-tZUh+Q*y(7_Gf1FEX71PXl`y1N#RMD~oXl8@V3nkAFa~z8`lUdb=N(>llIg z02fFB4(C1-!p>qoVNW5`38PB~;B77iGMIPJwi}DSF^p)9h(Qvyh;AzCFV;yU@HfWa!;wnR1M?|FNGG`F7#HC zYzfu0Cd5~aYY&)&K)yf7*V}7xKXUzrnRiWlVr7!eWPNN|Y&mQe;rTDED1yynFmJQO z*@in}Jf?t~&!hy>?#dz!9rRK^DY$EJOK`^UTB+uu@~J#cdP-7D91gsE0+ooovlG0W z6EG>hm7epi_RbRfVW_gaubk_!xej|bF=oOeES9xbj|uuC_hx#O!UTCBf0!JCV6a#EEAq`)g|GrUj+wkLr1Cdrbg5tgB_0 zC05B!Nff*|DvprfbZqm)LV@@*sTF;b z1k3x&TPqTjD^;u2f2qT@bAp;10>O7fPKC0eWg#P>1p8XGM*5oaadLmCdx||8GZ}*| zzmjq?Zlf;*S%KLYZL^WpYh~K_mv}BZmG)fg8}qD+S*CH|>~fdZEln?}U9!I9Olgyf z29*)cXy0mHA?d5Ms8;J17&1bb;1Bw)h9>&?K|?X$t`S7)teThLIx{JDimQ|V0cqL8 zdAlODG^Kb{;mU$~`Duka%l@iJxA*oIatr7t;C>yW?gZYVH=5a6Ub|ZNSeG93B-j$R zEuvH8u*k;|6T=4UTdAX^*U0wF5O<-ypLLb_p4njYJ9fJ_g8eRvdLfJvH4;aNWk4E- z5IeYT%st;0_cCXu<7;Jv^|tA5*^1)mymQ&(zYa-TlF~eNPIix?zm3s$2`g0u2hR#i zi!@b>tQm;i8TB!|AoO*}8~wPT|MW?ECTOg#i*`Twh9mer-aqV{jCpxyGcz+bWL*4O zJ-b^TUzTPo^z|c8OP8pdYr~)@;R^0#7@{)czDdxSVwHHYH?9L!?JXp zWq$}R`Ce7_FlClbEo*EXT;a5}^d=COgn5#4@)oex`zr#{9pa_JVzL1N)CCvM%XZf zi(MbRFY5P*Dn6>~p|wLM1Z`2jQWBz8fmybxWoHUzXRS#QB-c&p`PGoyr*L|O%>9IO z3VSOQS`o0>8}tW*n}oCp&J4L48C1P}4P$JZ8ZT=cs+JM9QCnB`fc)lrXkQCUsn}Rz z>}r|i*yIs{6XF_qT69y~K{87+PgdpgvMZmd=4)PSy1?d?Dtd_3g3f`nuKAV?rO^c; zxwF0wOP%uNQSy;bijSEel%EzP^-V4Ne$n)&t25s~u`Y_KRlnZB2Im^*HvJ6bEmQ4$ zWI~ujHDYDglf)s|t z;M2jsho~d+BHPAvs}>a7Kkj4oD^ayW?rGOTZP@NJI$|rkS({px+pG?wM<1xip5q7M zS@KBmOmKwC7g{BQlrOZh;9;SEhW!*)7M!SiD?dX!S#Vd`N1N}J-zv41bT64(v^DQ@ z&YH|E>B-3#Kh%HU<^AlB`Y*xXVvvWuO;o6J#dd5^RDWdM5w%6}WbB=YCZR)vM+Y|r zLy<`@(3VSA;jS>-DvGVy?=q%-os#)W)-T_ZN+#Nlxx@XY6U`=$sn4EL)Pk1-(xrBn(QieZws8`&QUL9 zv$Z#Z6N3Z6YeTvQw+K3`da2MT@?~A53&lHx2T3It=iBZ$U!gB)kheEyKxTPbS;~== zHfclBPNsEF|114w`tMouOBOl|Y#KdJyQJEk_>%ZJak?6f5f%C?@^a{d;K<--hI;zi z>R-hz_-a0}bA)*iw2@Edrxv~{dQ=ovy3lyiaToH zG@@t1vSX!P${v;1G(IQ~hDUIP`H*$B{kG>kbC-Uoc&!T#xgL5Uf{I!e85NNkdKsL5 z1)&uoNA-2IZcSVM2fbm8-UpAt=aruohvFW+h;F{+Z!NXRc zy9vzK&(I#o5^SaWi-$><$ww%Eh95OZ^_wC>K1B9VGGFWzHlaOGfIBYe#m{ChGJaSW zVX%>8%nopU<+Do6iAM_#2%eD>s9w}}DvX{_%V;mPot}lc?JZznmOk)LBa(y1R_{RQu-9Mk_{qOm>^st{4BHxuL18rqe}0W7SN3_ z^EfUzgn9R6*fUUICLdB4VCn5A z2UAm_{GCfqqo$%w#(n1&<-|3P2_I0AaB72m_rQ(GM%7n(=TzY z%3xpDq7`5c8A$h~Bj{A>7;;v-QbO1Zm*C-eNFIhd>lEaDw1d?mCqv=$h$L%}&B>o} zydcb+b1=dm7p#R9I~yK>-hz6t!mO}1j}uE^qcs5rwg9-vonUzfs)YN<9pUB!o2G<~ z^8okLXf_^p>|JI#=F2>0+#3Sj0?I(4{|PuYfARP7cZVv|0DRlzzvBM_^!{Mj>9v7= zZ_NJ7)`x8@gdO(;7U3$O^xMID$q@vhTvghe>oH5NYI9&_;Op>{!N9Edp@viAfUg+` zT>J0TEG%>Inu-?bfGewqK0_9~4#&t9sBZ_8E-!M@Cc)p-i& z?q$GVw}ef}0LS?d`1D_)ZKCFjxmQ3JZQ^F*nl|PlIhw0rlYwu)1{D7qSnosFmSD}} znIhN*hnP9cV5R}?Xb$-YnSosJ0WtwSm~qCylK8+#&^{+wCp2bG;(AWw?*csv1#mFF zjYpf83TnWXSxufMpOEROBS^qWF}QM~s2Hjiatd0ZzGEn`yCcV7KUCniP|Ja9n~M8! z0&3g}%u*ci`v-yHt3&$WY5D_pzY{3(d9armAU*d0CHX5}qkyWM3fz5t*ym69R^%nB z2Kk)mio0qU?5Vv34U1hRSWYwtOU-!UQ>fRP!M-yI9>C4)7w$2i%H8CuEXk8#hZqXZ z%RTV)e1y$=hKXbH0+kTOV118#fBS#-w}iYa{m%mvVFeoz4cx@N1eS9m zF_t_JRi72K82Up;=}(AMguvs|gKk6@(WB@c!iVq#ZA5>4MJEW0k)t$Pw1K`N+zf^H z|AY;MSLsGVk#Hq!&tbwz;=1BEFiE=UY4inZA234W5&5(e%>{o7y71Sy?wkx-EN=rl zkux#@>h!lgPu#OyHC>0n`{}aZu6$){50-&(6~c;{rp@4399G=C_)_tPvNQI@zGMDs z%y4KMKL<0!FT%F+y_#w|UcWb3r9Z11pdF%mB~yx9OS&VIy$yAV4`F}v*2|pWdWQL~_(B8A8HT?o94Tw4I-ywv*4@#%CEAXf98Iktzy40}Y(t{1uV%TT zNODQo0eTn=IfwcT3|LcWBo_trflK~wObfmPxrsUlPpA;c)E0akI|}z&+rV1iV9!bW zEz3jG;8I7SAirX}}S?M-^vS z9X=|eO}IRCyc{(;L~b$p2%KQIX%?y~*|Z&@6{$(ny&f?w0ZOl(9S6Y@s zS@fH&mF0-ZXZhVzlQhaN=~ft8gwzas87eVMfF+io^@8dAvy>LCCMPmsE`@Emb+}`a z-%l)}_ltWd64eJ)8|2Frm$hfX)LlCyJmjqYrCKZd1XY{rK=I}XCIXL~4WH!uK#b>^ zyMgzhuY-Rr`T*k@>=>zLaZhZ`eoytjfJBD<`%27LuDQp$;yg}YGE)cW+yjEQWE*NGd4|eC9~=tI zY9?QeS?jKCTWR{Q?Dyiqg*WnA<{nDl^kqoWzK`^~_V41qY%hAxl>{G%oltvW(^YLJ zH{Tt*C@eWF5Z+SXLB5V|#H_aaikIhPq%=vX$XZ##SVlN|GwIY+Nmp^Ac!jcUh%x$7 zY@@htHTJ|D3)v!1;u?B3y1KfyAmSEbOSk{(*lYXQ^2vJL-I>>m!;~5NrBU@`6Jvgl zSQ>F6YIsDE{;hh2s;^Qm-pEdN`Rs9y1ZS3Opw|n2o$q3mWS#hxzfDo(L{>i+RYQhfU8{2*5+t-B`C_*lz^9b($2)?FJu zQ9n)puQEsYfk0MSrL;u;?Q{B@2QdB;q998Z~KNhi%JORPdX`!jiblj}9Uvkg)wD+9%u4kVK{uG^2Y}H>2dmAw^>SlCNxWZ%8ABHxT*U7Fgn~aG!S1^t|^U;_p&VM7w48pgCw#^pFRn8RGeL6w!+b^EPqH zY|$0{OUc69d9A-YvMuQiQbvDy{pI%*QQDHP|9sCk4-t)x`cNmffvUx&cFyLFtCwrL zsw7aZT+E;F6G>~wLC{vKFe_`@Lrg%9Toa>2&MneAc`o6%njccniqP* zz$)iZalQ?+YMA$$<153wFPSuE95bJDkOVYf?1~LRn?kRJuMgi9);;*5_MNgT zkWVW`%js7@sW)WT26p&My&rwmIUVH^mPpnpd_Z69Rp!c$h}(fnWiQ(ttha;gH7ym! z>m`$s6P*36YwjOehtjDuNqT1b%ZwXeE#JnQV??(jo5hRkU1}QN=D)@dBL}HI$zAdi zdU&AJImLe5OqATnEzcN~d7z-a^}gp0=a=MyQ`?|_rTr^tGx~C3$g$x2`YzgBxt?t6 zp=?2xEtXrBXmIpjWvcwbzrn-&i=dq_Sz1$BPt(BgJoHP*Ed6xdWbGy}rBjN@ibQ#l z*hOwZeA)>upCoU;KsSC9^-wegO3or_igbi@s5nmCShS7`=Z$`sJF9Ylc~yD8(oaR6 zg57!3zir8$nURtjm-0{Qi?n~gO21XNHj}iCycBz@)=y0)HxH>(7TiL;P4!4AlcW)@ zKpW4yN_SaM-lVLHIh#tpSpRf?WABJrWeV7=yD8?YZ|VNh>q6Rv?Fso)*IzM1C}Dqh zS2{X5&o~>p8~bhtx-pXw1p(HWn@Y43c99-c7}XzjvxCm+4(s~q?1+f()h<()DjG-+ z(-iR+WAi`oTLU64MDT^0f|32ESSIBqGl1JU2%g-n^j%^R^N(kyqn33+#cE?>`Mc81 zMW^!yBVH};E@H&LDOPDI!=;dj&~w3qgPtn0gn#o|b`0YP zj11go8Y1JfHZ+m8dJp(4{&a9~E)$PX+O#`^xi~=6#SFotYAy+z?-UzJ2UE(3XgERV?Ii1#S6>G|TMT3DF{+u`N z`_r7-S$}8jPY+L5ryDXa7ESch@+JCT!oS99Y6r(|30bebrJJi?tE-{%N(KoNxkT4E z)4ZaDB47C`TZUu1cM|tIY=#}kFrulxlDmp2+F1RS;A6oQMwL|YMDWc%22%xn9*Ruy_ zJw5e*eOLbgLoM9hht6Qb4)(z04pP{ZJtK{`z*Wp7CZ*VkETiMGvhGzJh(<6#jEoc`{zP+?X~!ZY=%mqNmhyFdPz4)29R}6rG%oMVu0))u}D})kjw^i zU%CE7G+AFfNEQXPff(6qAQT?~4OdI?O*vmRMD<2-3B+1`L~p4}fzs7WzO7e1pAXJV)F^T+QsWE$u5>nZ6s}l#eMpT=u5?UHO($ccClal*i># zxz>W-w!wizyqB&}Uewl5^^>hq6snTcPqi(A(zH6wJk@!rM3CgS_!HR8P^kX~ZT3-w zKXBaZ@-*;v@E5XkiM8ZGVLM>+eibR`OCqNM zoj?Z(tBX{U7|Ca0Jb8wH&oxITQdE`p5iIaqTvsR+P7C}T(E48hgY=ng$G#1$@FhXL z<2tkgc30MS?6OBVE;uCi-Ic2L(fDZmxmbRv<`2-5&EV`*7ZZ(XbX~G+UYUkj=u@=< za+rZsN2(X*wm;}s+;Z(Kpbh*C7mbPfvHB4}*mW@!1A%u;zeC&U{|MBHY-!^f(1-dAU0>gyZ1dRp-)g~xQ9M+Wi6|*^I7~F{yY3~ zhH`d0hTw+dBp_D&4ks5>PQWtfNdX+pP-&P6m z#skz(N*NFOz3fAbP_{^E>IT%VpGg+lvfqW%;$bmX7_GblhViR(42<&*sA)GUZNNn@ zP@jrz%eYy>aPgtr;{NKsEu9dD3A>cuvI*bWOxz`W5Hci(bY5(rL!nqV&d#WjO6HX+>Y9jE6Ql}|jlzm{{?Ik*a0og+SE>9tN)QzLn z;lM9^!IbcYYRA4Y$qASi9)>>lJ*BOBRbC=D5Y1Qv^Q4YYdE6%VR~_Oy$w15yyQvy^ zGp6Q}+6q(tXl1K>46L3!dN;Zbb)aDQ6CA|9QN!7xlb(i&XsRmEm6Q(9-h2Z_Wei+I zAE-wuKkzfdFvm3!PHG%eLN!oBxp-nf*2i98d#+G#sP(1}ocJ4?-_ z$C34+^w*X$5H=`a+ytkSg*HGmc~<#CMd4`%sb2IH=^!-^*ENIc1rM9|3Qg=$Xt_H% zQ$9lW#q_izIUjq9QWwatiB0kZ^eNg%MmUfTBA}xQjrDJ8h!mtI60c>I94Y0JHD!?) zNm!(E%x)-9x(E$?eR< zVR*Qlgm=$9;T8NwJ;@*7*_Q)eWfxY@COJS|CXR>m9*Z^hkGj{b@FShSiA?2|l1V1w z>c&!5wKW|FH@Lg1RXIaep#w1G?hF3sEaf`!4bR=q?v%2j{#I4-B(D=&foV8ERwH^c z;Byl3)FC+wt1uf}niKR-C0V@y9L5WH=pIx%LAUrE)ZC7$GnC%qV7gFz0;jye%23)Z z-zWEjJNt)hEbpMaaWX!kMv8XY0PbfYvezBpJM1J?=@(E&BGDBbH5Qummw=zRrBnvG zV6(E2N>CD&nnXwC4p!xK;wyO`6Md10RR*d$s;iPf%_TmQ(drN~fU1x8XdpwBDb#mm zC3HcPsT}lu=1}jIbaYt;VbXq3)hRhpmMsAL(y0_;r;dgfY6o?<{1|iQM(PG|c&DSH zSxKna6Sd`3dZbjH4TQ%~GJ74qdliYj@^R`7c*4hkZV6G^!1rzjJcWnIVNh;9!$d$8 zA)WG8#t{pt5z1}aOMJ;>%e~NXO@sovw|o`xpr$;KUM+Ve+sS%#NcRp%Pq7Jd?D{P<1D9 zK#8RL2*=sFz&;ns+tmnao2(_qG0{p0JqXGf`_=RKxdWA9)F7!2GapfKCD~7TNVbrZ z!M4uFDG`n{!m1u4-YO2Jy!4j(2;G@fr5fRs*HV3yfpmGWM9A$;~FgdndEc%0Ck+)B`zWVmYNgi)uHrr zB?UamC^-&n@S*Bg;*_|HYbmxMB`R7RN#6o8WEIs#xm2ayT6>O@YhqOMGSwu|;Z3b_1sG6ISY1xE$QY&P!KzBKN(6+`h6rj+`nD zz`p6H^reFlOU>#5@U>ZVfY&QgN<#viUGfZ^K+@!Av`ab#U+H*?0E!N5c4Zb70!8TS z=s9UILE1syQF4jf(lo@FF2pHSMfLoIYKrK-L#cwVZ6Ke>-oP5(2HQT4&`AQmX9Lhx ziXxJgu+QJ9Rp9ZiBd#f@speuuT7#by2FCg&dW1ZR2$rX!>mLhb980Fl&?&|llTUt9 zK0_|#5%})}ddE<&mkfCF55SD~B{!=lh}xnt_3D3$i42qbGUuc@Xo_aBWfOQAbBO`SXfBi?#h@dwSFH@+>ITGDI9)fxcSaD! z`0i@NaYUa*#BR9{wOYCYR_swFfuNL;a3h#beu68;erUpED{aV&3YZpZ4{EPelk85M zR5GBaa2-27o{Ckg(etDXILZuEdm`5O!|kIg_$A9!lQN2WELLVbm5F3aJgfOi0_g*n z^iRsa>T|ri0l11mq>gF?1&Cuf{qu>d#2C3c)krLaZb5JIrTl_k1eb{@u>BunwSSg3 z5wE2Lwt;j-3E(~nUzv~cPvkJyrLNQyOyj!%&v6t_DF;#b1Th4<1|x}Pay_V}egvYg zP(4Pz1p*G9;pA612^7jB$bo7dXeQi%Mu7|HJYRA;)EFr0lNzpe0X{JfXT)P5D@H55 zs4IAf8!;oej1%A?Sqs_^(73@2AdawMCrrgUZ>}z-_Nd#TgkYj>O67=NRJ7cISPvva z4Jr{i_%td5v33M?QqI6RmV=)-2D8Ee&@~vUo~HV$9hFjGyJo0SP37POE$oY45b2* z$E<)x#0bRgNP0Y64(hPIv68xzQBbu$ihEj!>n~F_`9K({7s?dOvl^p=zD%7%=1Mnc zRnkzG=#OrmxeMgC2@XcDsJl`LRY}>4RliHg#!B)5!l8rO1-=0abx4Xq{250CP!*KQ z$lx-uhC)@Ke9`D`0376DMA@!ny5dK*B7@XJ&@5^oPNQqf!|-IfDJKwnpaR7-5iD3O zQ>DIiD3mMi!mSIs8R`(?9(KfYthOtNtQVlD5srx07VmPqnu%R=MQB~d#S)wFlsJH> zMK2SbQ~|iS-w`S5Qj)TsI;fl@c3@YeD#w6z9!WSPH+2pg3ZIDtr5o^5Dez4nsMv@m zaAE92`pONlejWko*_mvP9H1llM;-^=B^|KOeUy0O2eBKu;tL=E4-swQv*C+X6NeR% zt$dUx!h2;fCZE|}xq4&-uOb zh&rp3V}@W(Xjj5WSsts7B1R$~NW%9XLL9tEvg!^poh&=@)uYp_CE1J$V``cX5me?HO2 z<-Zsx&LczaCnrE>bGb5}S}E72BgNMwO*~NwX@>;ecj_1B^8XUg5xY`UADl5iutzE= zZJ0?_zp*SK5PYdmTLr?1YEp8Z|*_MgEeyk`3WL+*^IC+#=gk-IU+Na^Vj#l1_!b zNgCdOoty{s>M*&0`hcC3g$&z?cj%A3*_(}%!cYa}h^JA#{8E2Xd15Wh{UnRzrI=-wBbP{j!=E7zpV}8V8hv3;f0zyaJ6 zf~n7PEHuW}(C<)1#X))Bh!ZOS`YyBZyj;?4tmQ3I44o;rA^wu0sd3VKMEk|+OXeV| z5`u0g&m;25_rewKxI7d+wY~5ITu9%PDM}~B%c@KhHIz!wHE)SEI7_IGtGYp5V5(66 zDI3*N=>RhI`s7q%G+hf`43*$@KVBS9C6FIbu^8zl=$hnEsnD#u#EijIcssqwJ)AFw z5=uViwVi-y9-v&I-U@|;hvZE)QQolCh%@ppF4XBnZqP>|=%$2Dc|iF~n}O=sjk9DK z(S{YJpQK%(6j>fg_{(F6NcA=o0KYtCSCeg(ndCtFKI&zsaEp8<%_L)?gYkgADwk;?wT0J= zA8Mnw)OO^^r|2(YGLA@%5GYkc?c5!eYNR|H>W|%s_H<8q6tzx$DY0Y)r3cm79m(Bs=iuaMg?e&2nT9pH z0{?#IaRM zAoU_!RX$CRlPZyVJm1OW5_vn*TO1|Zg(aj}ct!t|lZZ2L>^hEKL@+XcAL23jO8i6K zCFWyB>_lbwg?_@`bZvm5Z!g-8Zio#khUGD9YfT-+nQ;?ZL=EMqh>c}xImPN~@(9xn zC&Mh#LcS#W$V-StApc*EDybB?;WchM^qf?R%tTyuPZwjKgx^Gej4syl&M0+*p7$odGN+OGqzPG992odvCvSyVPwp^>Uge5BgQ8`TZOBdoGmwLZ>{s${7InR6hwC8|_9 zNKXLw;VYSmzZHu(@&i%(1FD1HCt^&%rn83C=7THQRV{v+^A^s$B_J4@3E+8jUkf{iVzo&D2}e6P@T|h;=IY zLa~sCq%$~Sh7beka@ZwT<@4%l$*!!&xi$m(GV6#pV7p9GSHO`tR$N3r$0@c@xhMD$ zZ=qor3pDm}pi<1r6flm0DPMU#^`R3X zm?772X1G?mwmD8Yk6N!ecH0_Ysu^w{=-6OC;@a$-F-)9Vk4YkKifa{C zF7J$W+FUIhzbuvP#nv#}sgmi%euZ=LE9Onjjs&w=LWlLYHs5$vC+W{$i{JGg7-{bk91LxjVl@v90iR-o}E0(suT@?(yn-IGC7> z7Ow)2EYCErf`CTB<;!^obHTagLITHnuF~HFquj3b(lRXNw%)N#O9F-;2`E zX7tU<%~4&3#upPIR;}b(%rZ@fnf-5$OND(EM|AlJXnt!8rpnf)cD z(a-f?%72;ov(lfhnO;EGrsn^Xdnc!V?o-n|$2d8XziJ%rs{y*WdcYRHPk|8?EtQXj zI>WwH39P&{@SE3nV+$zme`I@r)vySab<+@n(GCaw1?#U4KBP3eV6#3t57v;U+Cq~plUy>cBts(pWyY`I7l;# zZN}|}YIqfOsvO}uXwh2SmQGfK?Shk(-a|v=40(Zh#hElw#22^`GK5#jA=( zZ-cSCb{k`eO(LWP1lQZ-gr!$@TRq|7FFBS~5q&Po_3#lYcH+@gz zUtn;w^;zZnD)@LvaHZ)LepXDXa1e9Kk)GWRvSxsGnywDd(jCYq>JxXkt)?x;;pHsn zz9NL8|2+k3!4wNqe9EGkbFWo1FH!gYx6RxM`T*q*QXa#3w*7G}Y*|&Gl)X;-eq0_ zHG`r9Y(ZNCNBihJf}tbuUUyQrO}7cNvwCzqkt$ww&2&EyZ-XJ815WH|)ZwSVK_jR_ z_OmufA7}`MUvd?&u$%Mu*-=avu!L$rsjh|cRNU{ZV{LEgTY3!q6J62XLQ7tU+(~(J z3)dHf7cD5dQS#K(*c=T9`-P4-?!8drize4F2F(`mJ~MSs4Glcsc<%JFd6_+5dAdD% zcm(N_wLNsnKq~*@ZgDl>x>-n!Ryr!4$~JUr^r$;IGL&MduHtbg;>`=C*L_0ZTtiV_OHi!*;+v z)-lL#vh}r{xA${wb{4o}q@VIWr6ux)y+n1KB?@o`(U>bX1Yh+xG`=%fa4M*yq(mlT3|WixT3z`+InqI;&Bad&X{b9Z&J34SB^ZY2P_ z$I^?zDn1Ifa2U|Wz2U&LlZ{}`guk1bGqmQ$l!604+^ll1tzpnwsx(mNE z!3qnaJkdGOVe*p=wd`Z4n4Y2kC;@5vMm`|Vk^@mScms1;DDD?qix+Wy^Mnz?c}%i5 z3vYzu!f7BcC7~_mm0GAtF9SNYRH`q}lil)J@Yi40|J(vja9+%++ugC;4kMIVn&~t#w^+wiXAuEyd zaHf6$kFzJRonz2VYlt~W3g&60=oYL}+oPxSFRp#Gv#b) zh%{ZYi>cu1@{%B45LrwkJAoH{Rca96A=(YC-H*q(ahFX0x z7_VD_5l+BczersMMmUus=pblWD^vk6-G_nfj)X5_FCc7d{~zc49q(&5(4q5yT#tl5 zaU1YrE0N{kw#R{CZb8507p6{eP+sniZk&MUbp`#dDY%OuqqHb3gp5fk~!O zDS;Mz0=SpaK=_^n=KH?<9!$^gG6k-2FX*1Xg+6vD@Zn`si=)67v*4e0gZlY(@XQ7D zS^mabC}SO8Cv9X2DUyER@iv2RM+>SI{%i}Zb_k|TB7EM`!CBpj8B%XN?doJ0x+)*g z^IL}L#sYNy`hv?_h>GVCW{(%Z#T^PI>u{_a7W_{WI=ACcV?3aURa-}CUPt1$x@h5qqWT=N=KsC~hgo`=={7wX^P z_(~gaYFp#2Hv~ew9^QJHzHh2J6O7p%;EsL(b~Xy!t=gFWbwZUNj-6AHcn5y)H_RFE z30NU*fk}?V)rBe+sEG$*WkkvZDzgx^01@dsl}2yJcNUW6zy%&btpd~d5T3+6;JD}G z`NR+oILGzWHq;o{jk-l}M6RQ)%p3xMkc4qWLUp5N zk*je({gfzaD7d0kge9(*_O7-E*4CERrdK7)i`NycE=(zSo_{m%curPk|Frih2me&d zdRhw1J$+U`&U>nNpw}*sIUcJ#Qhc`f=lIR`o#8oI*MW?YR=PulJh4FZf(LmyskdvP zqq?KH>zBZ)|I*oKd_T1@{mG&2R!cSerqN`zhe}WOsA1i zL^xL5CSnb9SMw8!Qo}W&{0`2On@dM4bHy3LVCQK|XlX*RwkSV8DK|d*Vs^P~clwx= zzkl`oG3e`-4{N>+EnT6b0uuv=g?5cN9y-Dp#hlY-dA{{7(0`;R3vcZQiZ7+!`0+e@ ztviHS%T?9w@`TEWz8bILM(X$Yx&mkW_4iz_>&I18lbqpp*>#^NPrKmSy_K5;G0Dc7 z#>RUAi4_`G0P@<$-{YGmg^s{V8pfUFM{!Q764VfK(0}VGy_UNG5%A14(|QljHnQYU zQEu@~tFOI{eXgmns6+nDtlZzNlN)}R{pQ)5(SNkGF0?|8_z3TsJF6}!XVpKY*AP~^ zk{0tMM_kd{?135oW$Z2NBSh$01;`aHSM2D&-biY5_z0ld3pk!XO?#@p-IeWT>k5Za zoJh_Bsx81Xz~i^^nP<9BTi@6IrGcG-n)_}req?t5=RXJRjc8y)^SDjy0dzenUcZ#FpL@M^zV>@xp_mA4 z)Zk9zlqP=~t_!c=aY;I7^)`kw$nUb&O2yz>nv=K0!T9y7f=dq){RYrvU7d`zY1>Nj|I@Qe4q>F4x$ z=VdbPG)yu~(tBw=pa8qpec3+CQr!}TJd`B;S&l1Vf6zTJbDk&kca5{pwhn;`jbG85 z{G0jvi)d4ok^#jt@~7l%PvyQ|eD&^g1)(TpS_7v3;QF)wYG2+#7nKZ1%S~5GnhA>1 zMH-+ylUn1698?&}%kz1;niX6459Ir+9SEvs-rI(Uv zaJ#(4|HqDFrVw^V8`O?%@?IAmv_!bR1MTvhsY9qP)|zf^Z#@G=yS;gx)0_VeO3ry* z{dHLODbLtiT;pF23+e|~f9Tg1;!#t07UM27V zyWs2V{aFJAXenL(O57rQpdQ{|YoWCR&OT&B))U5%r>`g!DP&p~avowP@2C1_*=d5GRSpINiFalbG%L z)n5KyV+1v-(DIaszvCVL}-JhkJVD8Mv)aN3lqf*s-QXS-giDFgvU(N)3Z(Ee@ zneDy3t~&u50Gr%TZ6{1O3;SgEOmF?CW=fq*=$-^jjd&mNPk7bPeSrhH7Uqo1lj+5| zpNo0xA=e^#40(#L3s0~a~2>q4u1*-26 zm(7LnCN_Xp)Th8L-glc^n_LA#NBM->mL}j~a+dliR~25`kC-D%_)^JI(=`bwtWk11 zslQ9L4aI8NUfQ6rT<-gfX{nD>#qHsZ;6|hNbD!cjOi!{ZsP!5D{c_^kJJLp znMCyK`XS$H!)mlz!xg>O@I^mKAF7kVaEXH>T@Jj0SCS!25Pwpap?|Jx%`52VACjK| z;kzEbAuXVwVMQKuTB(ZsFC4R&*3v+s6EyJN*k3q?I$yhv3){f~Ocu(yKLI^i*G}3F zTlE&9q@XyzI5J<8u|Ch&{e)|#?XSN8rtby6@wz2yZ|5KTa(j%UyPb0WkR}05Rf{?T zY|(q|NN|6PHD~$D*!{buT5uJz!+mFoSPOhe(RI%D(mdA`Z?0@xVE^JQ(;{C$mZ~Sj1D4MD z_tOJYF8zL;9$&mf)_E)nnpja&X?eM-9vbSUTd=$T++;odI!!cx3{FxW$dTr&1E?J4Jr@Kopo`j3>;kf)`bddJAEXfV-CJau zt>j)}n%D+8*lgx6%@XYuE(Wnfv>D8$r3IyN)>F3>F3CD1`AQ{imvnE zizT~9m_KHpOTU)cAg@8mUYCt&<(*ruN_qc)QbTw4nYu>&<%$Crmc|@@H~fqH5=X!f zd&9kh*YHvOcs;FM$Q1#-;1E}d9+-`PCT}xOd4`vv+aQQfUNV^e8>&A;*i&F0UZiT0Gl3DD3qIs> zWk0Hx%F=c*O6V=2hE_jf)?J@WR>q1Q?Dx%c%vq)bmV?%P_PW9^rH496jCDmimpHH5 z-kWz8b<54o))jYlWV)k;7SvvXoEL ze&HCtuI{|vLpz@KBjzhT?OEd?R@=Vov)VFp5RLvKPniQmK^9`Hshnv zvlyn`uLU z!%|WHObXQ+Q^AdxG?CIOVWoSAdmeDFX>eJNbsTlPck~8lC*2}AJ~^EB+Rh}`Z&$v3 zpLuuj;sP>1yr{XYR7xg?@~u5j`!@BSs7qkqA+ryc#zXbWPyCI({~%x`%8~Wi=FqFW zsa>Iq#l1xHPT&!HqQdGUKbMM>yELKk)}G=&@Lz#^HE}igg@}sdx%1q0wiT;mw;`XN z#`o1c(}>!|x^vpDTnICfT1gHAzw!lYsp*)e?pJ|klv!y!?(7hl(G%qg%3jP^E>q=c zJ6RKR`YmD&LFXRjS_co)RzfYIyG!d#!5#I18df0ushYcPI2PDXSQ9OG%oa0a{%IOx z-e%ruK5d=l=p=T)q;Dl_(Du+z)h2Qm=mav1FiBO!@xmkVqkIY~@C)ezK0@0HN~h6! zvo4r#2UKf5`3fE8=co2cN}5hx6E@5fs$HXL3C$1A3JNh?m>y{ zh${iwYK%yU6NP?m(%nNyfakz5q8nD)9Bu$(q#D3ab3J^wl88#wc$R?j>r(Axc%m)P z82NN|1J>I|FvSh*W@s60hHhF9us>OLH(P_N$SYhZdeA+n2yl=mV=k}`*pKzdc8`f9+ zGAYf3!r2zEa|=WbJj(p#j_?puFtd$OtiT7(mPUiQd{}hAZFq#7iAj1odR05nZ@D7w zfl6Bs(Fuj!`*NsK0}*~6D$*_FexMmzQ+0sns0df6MZn6as8eHT5mVPER1Gk6yFp8= zGrR%Hs{GHG3T9H>;16+_YEA3G!mdyKK^LGW@h|<1c!zsJ{{&sEMfm$wsG&F=Mo^D2 z`+WzL(_7%Eu96K27c!ojsKtMwqjMLSqej3eSH~Nugx+E!c+Zceu97aGzdFJ%zJ+oG z{%ZZ?by5Y?yXAq2^dN44-CO|A(^TLATcfMt3%>d^XnJKpx6Dty3eBkRN{+e*{S6BJ zgL*(oYtb8MM>K$&{dlMg9z<{F16IxhD5%+iIbbpKYYshuUFeZaAie_~cM3hu82CYX z5%I)m^?=-s_y#0fU(D?+n7+ruxy+wr(NWL=jnovg$4@{V%_BA{<;W=NA=!hvh$pp_ zOrajD?SMer2^7~d{GD#Z548gk2pn}V?vF%ekEaHd(N<{Wb^!wFEV@jWfD~JRiS89J zmlKrR=r1&Y`^7EX?<6?Yz^feHqmAlfB?|ogF6vr^gx;p7dKM?w2OymobOcCrHroN| zS(60kQoSi}$KQK|iCYjb1(*pFt<`%#UVVe(Nf1=IIzj(y2z33RI|ZcEZy-FXVp7`# zc%)bAPci|xmaXbs^e;kzOZft~og+X&G)M2}7tqXofR;@`U&uj}SFZqt6^|#f1bCI} z=(i*R$q|f6^aFJwo>WKhvpp~iCGgx=;5_SuNI49S1BVC~I!lLvTV07BXR;cLcN>SN z{0WoOLU7tw!;2@3T!bm;YM{@6cO{N00zT=5sseR79O{fUNdY*?7+^D+%blI_U3cfgHaAPnp5^oTWgw%tS4C6Da0< zII>JdM+kQT&l+G6p&+-2XrQ(so5PJL0!ltfn0vKX{eimxFC5tX2y_jugZVE3jTZ-2 z#NWV{&cK=|^K7g0e}y_0r|M?(Ue7B-i9~dlmZ68a9ecu#pSGQ-id~V29@Qv3gS)^p zJ;F~dP+WK_LkSV+@bl=s9aj#kncyBA!@G15Q}8ZW^Z?JJ>zAmu#pHPaCbOND-iRyS zz?Nx%jpznE=_$P7(|GR>%FqMAX}y=VLE#;}5u;cC$=kQee!!N7s{<45) zT#mW-b|4i4;QlxeyK)gak`3?{1EH4p5Pn2c;KTGU@QD%}lD+}8kcOC;gli~M+)T&2 zCn=Hy$Kvr6uN8m@U?g%3uw4qcse?QLr-xhl;z4i zpKilrX%OzdKd}_L1r^{pKZv*tgy0rrME|I_@tM^S>mEUK?ia9)G;lIz^*wN+#kh~> z$~xpD@30QMkOx1-Dm@E4pcPZ^$3$h|ET5^9@Mj!;LLixib+8pUnQ2&`Nx*mXP_g3> z2XoX2?9M()EOu-#6bRoU8pq+iUqI*m72H2#k%RR{lv#~lxy(iHAa)sx9%eYX3)q%x)NNc* zOIpeJNmDKu>+^#I=aFoDF}{)e*(Raq>5p{`ECTwf zQ9!KKM30pL3IPgZr)Kh9#p@eI#vGvxtP`WU%3x~Dso0ZLVzH%pMQMdIw+i|;Nw0WM)Lodw>m zJ9h3&^jaS%kFaY)5HIVHQxGfv1MXoZl=$wdj}fJ3L&>Hw_yE0t_=5SpT6oZ<2A+#NtynD@$aPl&3^P4`nFg(-Nq&>d-rHft4Aj+Hl6zP@aQp@xbZ^?+rc9hR{3oM@6y}Cmu;Q!(C}H zLE22!0?(%heFr`6Dsb|2Q%Up#_;wD4X3$bhJ3^q3vXHt5#g?Ub3a`|y@@TP0C~$8A z(={A=7YUM=)La}Wd;lZufuqC@M<{y`&>jiyf1n`M!*Rkk%i?Xen@`!^ILC|Elo8}R zdKz4IKSGCi7O)603`@dS4yrKI)P7j2IZSJA3N(CU^$Esd(3S4vG2C!Oy8?c;ouJcR zjo!d)V2AU4v^lzK`dK<&Lvd4>`Scnn;_iXdu#2w7tfMtR{@fO4pm*+cmvg;wu6K_| z9%dF6xqjRG+xA#rSfVW>EEg@QR@6t9J|*|_`{c~b?3n&L>#g|$UDIoHKw!lS;oZa8 zpaRY*7#*+8#YH^|dzCUyp0KeC`Dkthe^B4X-wO}ZI~ znX#&u^c*^x!^Ni3dnJ{+%^ySteMa+-=D7B`hT~dL*PyYnfXRk}i^@*XpYUAny~a1w zr@PlO{WY#4QCn!{+~RUWxi`u+2Rzq1$d3usiB;$VOcIiSm;6^;qhzkB ziZ$Eb+qukXvVAX}nBn=O;rFg-p62(gZ@J6Wyz6wWJw1Gm&waUN(aCi2_tHOSbIw|p z5To_$y%T-D!+B(Y$0%)eZYi6F`EYlrm-Lg5iUFc1y#-FcA+?;n!9UZih0}Hs=b~e% ze!$G`z*O?KDp5b#9Xi!(mj43(i};*j9s&GHIUbI`QBXoUCdCss=&I}{_A|SV$pkCr zDYz|L+_juuuKM8L?}OH}Nq8>35sIB!XMy9j?WZ}V^n7u@f_*v1GAE=S|KpYSpB(G^ zx$?wnRcn^i*jDMK=B)L2wttHB^Us)LT@`tv$CQoT1`)rPaEgjKhDv_o`lR=&}fH0wQbd^-n}_#1tTjFH?B zU|45@H<>DqROi#@pdaPWrtqKm%FI*nK8)}~pWs^MIOw2V4RMDic_mbI^ianM5$YpB zo@lvLdZDmw!OX%ng(GwG(;aDs{3zjsXF%mH)rUo_`>RFh3=dDo-Hf=OvwqG?X_0NW zWY9rAdBKCrZw!3nQ{>TDE0TRNy<&kyy(J$Ohe%}z?XF~9su}xLV}c|2ZOrnWlm`(H zrTP1ALaautW(Q~=8#;O~@%zWm=yTd*oaPKMO?11bx<^6TJdkX{9pzKGXnvn|o#qPD z6l|#+ah&s}&0-61xE-woPcRAnq43oc=xDw3xA}Hy`;vd5+a3##6ggj?J1cv2?o0C| zCOvR}c;$$-5#_4-1$`%*7Eb!z{+r|5^pq<3O@s{nmY{x>?o~Wju7U45{aNq^Be2VM z2?NC^LMO4id{nIj^XP^0Kh>y7$98kyZj?v}>6nmA=wlyse}s~zDzFQ8@6_kg!P zKMeQS8HzV5T0hrjp`TKlUd2c1>ghYf=O>0KB}(Kx@tixuIo{FFG1}SGeNgNp&xN*a z1>p40%ONg}rB2cCy#2XZdD{wR7rig}X6k7=Tq2bWci!PD2LOW-a1gQP{&nF0lV8K+C5w~!9?E|ToUen5FhHeWRQqjY^A^Fp^ChJMctduI5U5nocX_|1q?!ha{Uk!TY zJIlC}-Kg{tF1aeZ6lbxkt+-0bCZ^Hf*}j@h+V|Q|{2!(!xexAs7Ff`;E7pPnTOi6oH)bT*w);XS+GBs%pVHRodnY;TZE&H{Xe^hl*#@!ZuD&*5FL=_ z)t2uh@z$q>C-SrfFG^0@qm&Ms`JTy8@Eqhd)!0hglxYKQ&~kYg@{Fx;vfc>o{m=RyGBhS%+T`L#`5yT-3cStJgdDn??xe?h@0;FR zyc!r1`IdB5at7qGFC!~TfY0`FbOHV*_L1Wl6Sqk-MN^wU!hV73#2IKnoK^=?+qh%; z5YHITsW|OQ^?Ue@6ns^Y1*XfpfDRst$ePTCvsv_WT>k@1!p+J)e%_G0_@YkMQvyMU>S}pLdhRqd11fW-wh?y8tBLYWr!g@Z-4~+#BvPTZ8@%zJELzu^y;Me&ci| zz!^V5e*v4`4i&ZjM27T8j1#*FF|G;F)Xzb@Z3W%EM8{@FRmVu%AG57=OY!Q$tlUqT zt24v$hg&1aK(E(94d7EHBudPlRS~xeTS?3E!vkF3T&E5C~=5X>OubzKvo`=y(2IO$>ZP_-*rGx})vQ62vw z8Z&OLo+g?QOLa)pmMxIr9=5~ zEp)whp#~@H^VXIxsT$S86T^Ai^tr@0(l2ZjgbB<>x=EUZD zAxtUXL-$Gl z*0>O;yc7Br{7puHv*cIg{%-0WQ%T*6O-v!bh#r@3mn5ianM7GmTG%oe%XuPa@%=RXBN-crd*x?>2cq1cP ziJqDt`V}TP>}+RF^RJQXZ!!EfW*Tpx&mGG@;Y{pJwk{XUIoW1#&RR}21+Hob)eEYc z`{`)z71o=dp})QWY8<<9YBb>|@FOs3d4))@0;s1g-~`T9Q&21a4HUz8pn_^3PDM$> zMFCmx0QVj!mJhH6SUX!Z=A6>SzyUNaVhTs)f6JYlGbg8Q!B0y~T&R&1DMU@iCXl1s1M5oIe=MV?Cf;*bVyk$DuCS9%`#qfk|AC zNy8{|3A&F9m652Do1i0c7xpqG&2(7l|CUQ;!-6%=1CNX?;a zx*1-qy9A@)kLg_v_;J>?Z?b)}HnLLI9@f3q2AIm;H~E=1n5vrpGpAe1+0tx9aLV#^ zA9B|gCWs$_o@!3)B(G5dJsokXSAqM^9%Fm6G?XLo3;{=UaHnsJ6zoRo1PT@!T<2u>50L zYV_&V+q_vCB%p0Y#%6jR%MI9{czD`Q`Ilfr)`0~HVzfuO6>m= z*!}51EA{}gj+VP&mAw!mghhzLd!0p&X^sSYlFeXiX*~_B@qA03Wt{b{b)mJ1HN@K6 zTF=(T9^ueB!(Gwt%|aI-+U`la<>TndRDionGrA4q|9@@Z!*GY(f$1T^r{HZE`PRT5 z9|wkF0N&X!?2%!ZW~BmQRg3QpKcdUr75w)`ZZtQ8TY>-E2ybN`Cgg5*A3L6HjGb2% znBK~GLl=Mwu+uBhvsp^jMI1R$=4_4`PX>B~3KWnp!=GZ1d2iX?FF`HBaqC60B z`6eb!6`3673|pVug2$X{@<)L&p(_r@%=Bb&yy!d)%I?><~-ZUdr3 zG`AFgZs4kNW&emQ#dYPj;(b2iu5sJoada8GrYqiKB<{NrHxQmj_t_0g2ra z3bNj~$1o_AcEWD*g32??q+%rwg!1ki%R?iAOJCoM5u5@l`Vd*&2YSS2#RO(@hG7T|DTK=`>+uA$sI;*>< z2+PF1(sX%`;zg_^pTnIq2NAg~_a0Nvde}FE`Tp2j^TFuK!%A}l1-}cp8y~(RT$|nk z`D5l@V6TNDR!i88eK;L=6X^1DP>Xtt+36x^MzvuJ@wRqjy7-v63#Xzr(ANd_0l%-} z)anVZo2JM*Z=)+$4a#&z>S005GIawIw9_L!+Omy6_U$C9F=2#k8 z!pvT#9;IJO#+Hc1BTH(P{!==$ba82*>A7jCnXwkzW+4t11Cx^>ddhu}3tE7$Xv4IF z`)M`!jz(#6H6m2JN;QR=EX`BRbd64v$e-n}0CS+{v$6Ba^R@Y2{8+@kdzj@{=NK*@ zdcPfjk{)%g50#Y`S&Y@}C)m==FX|^eQi?y>A;7fCc zZNb{0N#1}t1ysEb)6_7`;yz*4SsOY|wdFGOtjYZnIngQHWo3H|?sK2zxjDgfxHPq- zN6F9P%;LxrqO@0O|5Cn`D$OaqWLj-5wOq2f?X8^#_eS7W9wS?RO&q1>Gruv>?!^z+ zG}YeGw%6Uz_0xUT=4*p>&$T18b+x&g*_r}G@@Tx*aJhH>|+==Fu&R8Ti2t@!)?r;eS{bp3|E;C%m}RR{}4Z~vIn?E{4ucT z+VeNKTWlPZ-=;vXd>!4H62W4v4upCYWt3bGJ|=;#qmEXNeaQcEEFVniCGX&)*&Z&U zZwsF0^~l+ly*JmlKq>SsX;QkY#0Y0S4|{Ealnx48-S5Olsun1b5a7<5Bf5Xkr0Bd1 zsrvn36*kk^H1jo$wWHu=8^e?ANTwCr1iMtvRlt8&WD1!-I0451;W{1b?*o&7)p&wQ z1A>Obud49&S`8JfWr&C`Fmc?C8N_C_Cg#W8=t;~EIIVqTmUI5v3c7cir+lpDhVHID zRNqH83VyXybl6E-knJ)$R@!L$B6~gC7W0JC z^Cgib>BTRLUKJ$doy)zK=a06rL@e>F%v2Ns1FXgG4eL#XV*T z{<=aX<6Hn&7Brz^s#)<$r@JcP-TjMe)O!PkCZuVuHpINiKo*=ujpbr{g*H z7vEXuqt}-?^XYFIUK-AU&G`;~y>6Wr{dJRO0$+p(*NzRQ*J8ezS~d%llf{!lHZ=5O zT{(_JP`OS+wm-rWVQy?bWg<$K6!nE3?G`YHYF2TF_s?gDF#P?%I zKu5nGy64xKhSWI&xo4fFU&&r)f!?n*X3Z$-1h*r zaZ(m#LGgfM`Eu0l?U*FgKR3Ab+KT!&I-~BBZj+&^ak$~G{uP`9XX&j_B29-^ax!A# z3NDRBUI%={Rm|WA2u_CjgUj!#U4!#NOHNXOk?wENSL=leY9D`WDBE8Wznj z%+DW^XUL1rpH@(mKeAv~$-mZ_)&nJ}rRBs>dV{R72fD|xXLU`u8q`|$s4h|;she(S z932pM-nEb>6*{I4hP#N69F3ksb;4mh~HZabc0X^}7o`F0!ny;gor){B| zrW>hWVAy0>s2{ECqicW_!Rgj%9z$<-7azn|=YBC8|DOw%X?P?nKjm-IUDOvRf!K+2 z4R*eP>flcMbDUAzfKoD;hnQ|aTe4 zjOGoZz%kTezrm(S0K2+}qC$H=207zN}KeQe@@tZw3T^(izSh^Ps zL-qR&D~-E!P53bFFT=n3KA3jB;ac<1DagAPX?y@i>K zQ`beWX0{<>d6UDS6R-@b7Ds`axChqID=1m~k#B;lbp_fL-JvFT5B}Me+40kHcFF%3}+wm8&7ruSZ`%sgq5MV1iY6#9lfb= zag}h{odJDQPiIHRZoAF)%6h}{$XwMtz*JI_T70f}X0cj$Cx2W1vcl~}O|e7tg%^wA z7v-GmT4>ERrMnB+!(1UKkoBpmaEi|1_OgU7%uq+a(lFWMmp&1>$|^l=EU%eKcL(3H zAvp1CsVU5C`XN?*2%}-g(oLwD^g5j4KJd`Ig1r$5kBxtTrA4M?66 zR55i?n-$4J#OP_9c3c-VPJ^~7w<#Qa>-FfZEYsB3>}HDm60(cIs3 ztkhi+RMMjOaZ#(H9R=U>-34()LyNW-o+^?`I-8f;BaqR4wo#5m;+^IjGZdNJGCs{1 zq<;l>^Jcmlx>vf(#(u`T{1>_!pP_G~8^ffLUuc>=Prt{VH>ca7CsX=AqP_z>%HsL| z-s^X{TrSC__d-Y^v?P>(6eGP#?_VjRh=9_oq98@-il72g6zN5Jl@@A%03jhHA-!HM z*L(Ya=H4IAKksw7_uhBQ?Ck8!?CkCguU?SDAA?gtlVBx@0gPP)1NbKpuc8X_e<}7t zi7`*eL1e#Mh>csjxw-+@iVa|_6B>hqkL{kngik#o80g4MjD9o zAy3S4=BK8!MvYfMP&A!Kn{BG@yF^YyLSz+|0&_#0lI)@Ja#3MI>`!aVFc;+14M#=(N2u%5Iiu}eI}IMk46$k%_bYp0#8 z+1}#Q@>snN+QTl@Jk`!--jm2mWkNEFc~|xB=A3#j+WGNe7G|BnAnP0 zO@&|wbhN|b%iznm5ry~zNt!rM)JHs4T#Pw8U$jSf9}#+{3wFRyF`s(_JD`5x*xaXJ z-Jj&T3R{~Jc7uBC`ZQUtnYWvs8IK!Z8|LDy_j>JO?L0)w_($DUJ-x-yvP3;bK8OINNQUe@nwjQ-A2RemG%lM%2U`uvHC*jU${_!^`2%z^Y6u_%}U)IHymU zRhX3x#C)2foIHeA9h@p>Hb81l;3_dA_zT?km7uk7w@?82SRlG9x-6222Z#k||6OR| zNuv9(R#XcEg>}#`^Y}de?}$xQ3X9NG&nf649pEuN$I-#Tu^)zifd;d$`Cn71ah~D1 zPNOZ>?$JhTAGDlOPgB2EpHS~>cB@7=uTaNnj%$zUJDEh_D2P66o8=k>A1EGI38w=h zjKY68W~n2E1I4{CGngQGAs#PD#m?U@l5EU^z7fR$CO=VU#53Vw=IV>sys(~vB6tD; z<}mvZA5;M-Ph%eQJ>tWNAYo?XwCZ$N8Y20XeagUyQL4jGe+;p<@q$uGtlyfR9Yn_JF3jB^og}yQ(iY=QO7@55(#F zFm>COcG_?Ba^nnRiSdcK#5x4AZ~kyTbcJI_mzJx%~^lDA>s=y}Oj*QS#z1iZoPzE7!G*)`Hp$Zoz5G<55-=b zMDZ)}canN=N+k!pczV)IV5pXRVOTz^Hs0JhTS<|)=Yw%;6qh%0XP*qE+}%(H{v z9vps^_#^nl86!O-&GzX4?QEfBB%+%CC{7eJ;#<(@|AZx*FItEGS%$u-6}Vup*aItv z1A4$Y*wvPT(%*v02XM!5zr+qNf7p1VafbZ>w*X^bfm0hzyv5*Kez4J9f#ry-mt4&M zzr+Yi6_p^4wpK7-@EUfyEZ#8qz*fR9@>KVB*EMIZf%^ui0!qiKfWzdD~s#l2K8l;WXD-Ba&No_QJZYi;@v&)_7Si|1WNx+)sF6=<~ z0-k$%VD5huUTqFZ-O|m{M#&BI?mO{#aW=d~6o^iUc8L0ju49gP4wlkzVK-P#Il{le z6W=2qSmj+nX*H!NAUQzmzx0IZ4xvK z%SH9#-4eZIk+fR6QOcF(A-cgF#4hNA(Q6f{z*)~>4PqrM{M}*c{tDLN39vZ6gEg)g zc8|gQ82&TF?)nY3#Ut z9@t`wp&jL7B&^50T86d0ui+ngsVmp1cGTG~+74O=SzehA8V?x$)c>g4r~N~-x5cHN zq<-8yviY~>+3Kbit@Z=`S-n#~-x!W~G9}jb_C?qkcmh$KK7vOvB_hXv1lb>RZ^yNJk^#A^d;&cYe@CD{L$j8jK5VZVEgcu+%Ofo%`lD|i)R4z9!~AIbd@ z7Gw$bPIkqPyMxY|ShfESR`Cpr(R9~%*6^$TxbC*LO5<$l(ef0MXRi8|`fbZ^T959G zUSx@&@eT!}%Q8 z6&HZ^Fn1b0g++A*9evjO7U4--t$ViE+VSLK&<)(U^(lISU+r+IOSp zHd>~dH^J^W&9GK))aB@QfkH=Va$D}TOw~-*zR+h6FN^aRxvo(t>wnqs4Ni|mensr|3j~3 z(+(G`Ck+)Ph>l?_trDhyN-sjvwt}yZcd*Qj=DRWSvLGE2pn0{x_rNvob8ZP&i`DV# zxPOEv|AQyRdx)ba#)$+C&K?}XsQb+IBeaIYkenK8igl)Cq`5z=;*Si$hW`2`7<)6c zBedJKM(t5ui~glyzEKOV^2S^NZ(@t=;~ed62Nu>N<&X~eI<_4a}SOVD4Zge}4`qIu9AR--2up>0X93vGb5)e$!2r-J?9 zhVg=2tbi85C(9DF&4S$(_i?^vK71?anM>Gb|1q@HSJ?Nfz!{rGSabT^{l=B!+J^PO zRL2r~qpg)~BI3qLVC&yy>S(%Zj4-Y=#2KFI*Xif#A0SRuyfMY3MP!r)bCZQ<6PV@7Ih4r#S!Z$(#bZ-mn{T%ETSc3LfqeTM+ zbtof^pAR{p&j(n6SO%WF6SJlj_`Zhz@?gFh z2>X3EVGp!!0OZ{RuG2y%vf0Sf8w=8e4ml@AHiC! z0{iRN;v8O_`!OsWa_Cq+ILsOtw_}NLae_?OBd(4iolIZNPe9PuCS!XLxt%42zNkQ5lZnEJ+G<EB~BUK?@Yj5}o_!+Z{SLi)E)=D`aX& zt3I%5@4-%h815716ZFnWoS2Bfm|2S6kzh~!UW}X%uwUtA@o8SF|&KLxt#oiChs zoqr)(>pbia9DyAJY1r@8!#M!=)0_+O8HoFt&ZUT=`OsP7tcAa#AXgf+@xiWzuAPW3 zs>f-&{_weT9X{X_u+M&*=NWe9bF3v#6p$(4(KiUINOz^S|&pA`D>bf0!=O2M< z+2M~V9Go};9*e@^70Ax5gWh$3XmF_#(arZk;HqOO~!M@l}-OZily*j$v;A)G})(uoT3cCs>!OPJy zSh)AV)6NCha-Jg6NG(pVD6#J#5!U~9I1MxiJ3{8*M8Pij1G^0lSO6;rooy6gA5#oQ zaSBfSbiocFI)ywLr$c6QR>JDG%}gKJak^u%E@8=mtLj z;M$K=f8a#V2AsxNj@>}hV0#`8|3jUyYakw{4rK6?V}b|j27JnK(jp({N$4cTEqE%r z37GEUjPqGsr$M)eadPSpTsr{oAGoI{C-LS2(pVovlQ>!DByz9CgM8`Fo)njl@$Q! zrO+9qxJ2F$*b3Iy8T1>6!{8FY_D6R-?-M6tvd~)wZ|(3*Zij_cjq{6YTn6|pvU;CU z4maMp5u=rd^5~M_5}^$GX6uvT44ncWs+;~R0hb&Q(fM1rDa0oTr$NK;D+b>Y_(Y+d zvAE*BcTxX$pM+K?dY@9dM5GMEH|2`JMd=ffgWj=MtoJRI7vaSNl}cElda29^{7&;; zi6}W4SDF{MY3T7xoB^YAV085cRnh<6@WIprpWavn&VfCCFs^~#&(Qz7%fa&jcu(Iw z@lF1ox_I+a`tIIenJ7CGcQ_x4-lM)u_x56nw?9+RrzzeRy~jlmASOznGZlnK1+dF_ z`;3SGFx}&_dwWZO=hTlpFW%)yPZ!~p{ZPy4E*3R)KpQgv2m5K~{oT_GZx8Q(qWMg; zwzn5r9zH8z+8-{RsYwbT=0pD z;1>l*rvV%lDDyh_!DT!>34ZVlZN3F;6yhqv$Bq$3H8!EFM}Qd-kZ^vG`ZfeO4?v4( zTv`zgIt(L=_y>(VDPY!kv0Q=@%TRk2pdhT?21HZ`T{VE<0qo=#z@7dCpCV4R7yp0t z-fzJ@ajMf^c&M-Py&Q_pIv&J{!OLFGMV{-50G$f&%26Ut8UrE&&RPz{7)SwTO?XQD z%?&SQNq}1hC<%_sz%b<`?8&{99R$b?UJN?HPvEH-?axP>?*ej4XGU)*(Y89kL{Oc> zTN6^+(dq`Y@-kM${{X}%(SpBFdKGYg1hD>wQ^N~zs_z!cD?r^>!HYLTRuL|W0C7Cv zqESUSD*;XeSf^qL#(R$)XR1XY~|ObvjBXuAS5b`Y&=z%RmKIbh(R4&wX)X!|3K5EHP?1Fz`; zyd|SYNMfXcy2zK_Skz(#UN3+`lJSf=AfF)E9tSh#u@nkEmy8 zL2sNP9*n%RpvioVcG1|V!Z=Do**(3fiFVweC(;ENocAZLLY$Sj0QCS#$z+t(6YcH^ z&fOC&BMRoA^;7UG4e%r*RVUEn9Q5a6)RlwZvC!z20~?>9)t?|u7$Eu@_?rc6rU7qK z&`&1H7?1b!kRuu2f8g9^HcBTArV5yopzc`I-2tiDKL&4EE}+8*vEmHx4D>}9V44E> zNZRD!R~cxw3T;vX6DrWI4m+Hu0bfzTr3nEnBRf1 zZN_P21=`#PFoXbG*+`R!`ze5RG=8NaKaGcMJoyAYLQ(#L05QcCh(`~TUrv(nL%k9` z6D5(Z)dP?qY86t1z_Z2!^m+hl?th)>2%2`_a>ypO~6TKb`sGioxFU>44zYmj~nF@AAISR zYIcmUY_yAbNl(zqH1Liz@5mGZ7Se!7Lm*A05O9*dRfBxD@PssF9ZDnK$^f&3a~aY{ z;Qo~v*g62shPW!(e&A><>fL0JEX1Q~sH%}O#C3!<_rI~>pr4g+(0jkTWs|Jvj0Z%01 ztGxVC2N-VxLIG;6fF!sM=!hc{%?MB$VZ1x2M~V`0mL0uAJ6{68+X_+kK#Y_b_+5h$ zeh@S;16+xC25CG2piDX&EI=sDzmfpov2~ra+ zR{q?ii*nTI;Ci5Tfg9*AI9A)M3fkJZdbrzFn~G>0c` zP7TP(V=bbEA}2|GqoL=r@Mv25@-N*w^Cb%a_L3$MFe8p%VwTiSN z5#UcmONchhy<>*72nq6%&RT*?gTA5}3DGo-ew`PWCOjuCK?%ATjDC;t`WQf{1@sY1 zYcvmk2|SSAK%-8Byb^pG`;oU)q8-vySzGw`}P^t*03FY4g zUUVqy0P>Q>#)%dapKzm0(pHq9oFMdB9kBEWP(Q+R1t=y4J^28BeaJIK0BWGr0;D8u zk|;xfT1d+#?waczkygM&)(PV58bHZfA*lRz-qEN4bZX$9;=8$lj|;#KJIlE3#WnG4 zD_VRPAL4~H$GnaDZvx&Jw66g0TVpHNpgi|+3-h$)MUkwAVm$zB8ycyE|NjSYf1j|u)Fpp=*xuuBE5n* z9a+iDfSqOmN^sQ_Jfqn;&GgIBGLk-QvSg*;0rFTtO>=9q2(dh#I0tDtYV>WBmtPA&X9h@}k*FmZpYiCyWaR6Fp6!8J zh!c}0OZKruJnaX_Sq)SUIQsz?gnhE1$^nlYIH%c>8Lf}?QVe05tSbVfN9yro=)oF|(c&1;A=QwgMH5{^m6Q%PjcB>lGxzo|AVi|lm8CTpqCl;fV{J82ap#i%{h2GZ6V@RoFNx~OzoXCa)iR=^_sCLEK+ zn8slh(vVI;c3Z+O)pHw?^a<|N-rA^7NZyg8C1_~&cN_W1K0-MNLk6^sF2WP($7DaZ z;yF<~YtN;7>H(_F0Enq~MW7d&fTvfp$^We`sy3ZwKi3#ZgHeVuMXP?q6UkB+fxPUx8M|IaaImXlB#Ve^k#r^O z5k5%6AjwawV(}=2rz0)dW2qP7 z(cVndNb8Yo&k*$N>Kv_#vN=d!BFwW^Ahy*+b7WT~?T{?oodA1=_bvn9WIf2lvku_u7xabor&4UD)lSzM?*s*B_p{bwx&#H&ey zl58cp$7%}1g^80>ZdwOqIXTH#dc&@Gl2$`!Bg=7Bc+*lkRsylGk%XnSN`jH35w(Zf zOPT^fLhYe){|U>~Hr8H2PYL4& z;FbD_T^XQz;=goJ?-9>u*RQE3DJLtB2}Y{-J*KDyggL5-zUd8%CBit#5rUIGEVQgN zBWcXaGfGd-SRLd&rb%*)2GH(`O=Prq4QP-+&t)JLSBl%YSUuh@Q}{vkQZ+NP;A zR!oK@3-vYSq1xWx5j-V;mneya zocfQ&8MT~xntF=G8QW*?`;HwAl!7o$zXw6i+N*9{>(5NGc z&F=SjFS%Ln&W>K<(<~3Au}r*_<~$@T>5iVTcVtncJCZ%`IXv+Vk}@n8Cdos2*l%{% z0V^jc2bD>1(H$#0-q%2Nuus|gXcq*%Q=oKqM+E(1KO_@K0=$>5EcYc&NSur1bnkbm z&^K{SHV<)3;?$Iz_}+W@LciHE+5f~Z={@CY)~|ZCAuNY9*uRPEcT*%mIjHs=$^O`OKU`R>U3roE$1(P+J2%Lt#P3<||GpPl zDa1+*Rtga}XXi~Uw+mixY+AF-7Go|?&?L3B>ICh8{HN#l#|XtxN_Fzp$j(Mjta z)PF3MQNNQOM081%OSHg_YN9!M#?lL`QB$uIUC`P@5=jke2}uGsW{zy> zR2%VYTCt4x+k@ixHLO=Qvhayi$)3${|P4I2UI4_ ziD;bCAB{xPAgOeAtP=+4k6KG>3zUNgD;VwcA!{LNJ7hm%bsC}sY9nznf}MJWxCPnh zNg~qDE!uBJtt5LgTO0K!;h%b$_#N>sqAluUqDXqLf=VOCW5omQ6?CBVY=T5*n1rX4x#X#6})3L9{jg_$=&VJ-P z2@6j@yrI4(&wj*>q^JYYBY>k!K=aJYiD-?HcFFKKKfs#S4&}fr0@^qjEh76d+50=; z?Mv)O+6M@>z$#b+?~aM^**KUJ2ycI1K#$ZSLInrwfa?((bqnf1+mKoWxSk=>$Tm>M zdidi!2rO*igy9qm#h2-iQ-uM5>>VQK%tpNvv1|G?TGI#cPX;Esb6%t6*Wn%IFQg7Z ze-r~ZUGNdW%fdX+!fWh99}f(cpwF^F7a8z8I0caYf;<iK@9{Y4 z{hZm#jp03jS0I1EIN>#%S2Bx+;=H#DCl4#&b?BmGg=DGZkYpI*&3`0`5H|^vaSn98 za2w9|v=UVC=HpD#A)FYQ0WU0{qn#(*`Oa#1+nV7_a^@fo?@#va_M7%f+eh}Hw&T_n z@RX2e`3q+!Y7pP?glU5{!E)ZX(HsVEW%Z_g#wjMLd7XKXWwxo-)Y;B=d~b7@E?ZW+ zV>#`dFYUWL-}BZn_c^2ar$yz6O1@S#?!*JrDwyDY(vFR$`x=Q~RNv#&@t&L>+I zAbTW@l?cQa#gUR!aTdJE%ogYcNBLjiOcKL)VwZjb_U-@9J&C^Aj~)M7jFWFMB9`Mr zHpC@}7?OZDEB`+KGr@3v z0`C^iXLI16sV_qjZaOj{I7RUr&M)u83C6*oB19ws9j4=c8fOj8354Q&RfX%g^Q`l= z^Ma$&mSQu)i}X0lMN^`&n|X%uJ6()^iROrAg6_Qbfcla;M3-#Ps$-g_X^)xb7)#X~ zv_mb|?XNAJ&0}31czWXMVT;+Unr1>1cE(?sx0Dn#<}V9%u%Kd?+AAe_{eJR7~Vxj zgs49E;gN6|;xrjK_Yg^B2+mVI#2EV9H4W#(4Dg5E-~N{ck>|`K;fuwi3)COdP18l| zmZ~N+rL=6)7*u;28=LRwerdT{U)#tx?6oMhN>v;ERY$t>BlCLeCGHi$$2hBzD~|J< zFSkh_%LW9D40@$(9dOM*RhbgBN%vNw12SQ&%V|22){)>J;Z6^Nz&btk-|Kj zzEbmJ5R;%0K7$lE7c&xZk}5cz5J4gqd@}{n6h6X9u%n17)tgCVq7WBEixVyW@YdJH zaRO)6Y8)4>)w)ki26KU-gZ5AhU-PI%s9xPv+4!*GtHut^C$&8Fq^i=|1%^1wG_|XC zmbT6@&gRrxObNWH!tdeBaG7MAqC$pK@bYlLtqc4^d0u%bxL2@7c{Xrn-~i>R0E6FZ z`7FhC#gD$@Mfb&wEZ-+s^grGLoEYmX$m5m3x5^+u?O<**mv9;f`_GuJ@BsQZxZq^O z$?D5wa-PGp^-qXQvEG^CP&+2rH(7E_z04N#VM8ZvzJ}43s)wqcHMFaT$JE-fHJth& zbwAbjRZ}Wdsw0LgEx*@pZ|q}PWPfgoG{w2QiWc(!;v|U11Z)euYq{_2;u>LanlJFS`1BMY{-Td2 zbXsVkUxQ!wh(QV4B2R}ThDAr8kKG%d8vJ3%oUpT@&;1kp{`P+o_`UqJc&qSB=}H-V z#=xh-eZltv0dveV6p_XAa%db**)2+Jn_8l?OFh_C(tvL!)(? zz=kNkBSiiI6T)T(Mf*<)J`}Yu@z?n9@TcL)F@GeCirpF3BSaYSZ}?MM=n#_Wcl>bF!+tK`0mFWXT3xcWoY;rbs6X6OIWq%n_ge)+bh{Bz@4 z*JVSoX1Xg;c3<$fYnZf8L}v77L2u=sg&SL)Xx%;GaOAlOhmL#l+afjE3=|X)& z76uIqHU%z`=Zn(BvwXkvtr7{jhls9=Uu+w0 zS#7?H_{!t-UuvdlY)w5IBkK3orPQX^3@QKUZNH*_D_1lvS6wMwo?lxt+^E)$diQ5} zUCTK8MN@!wyrZX-FZ$YJ<&6tm7gZe5MJZ5riyqawZ)!mdHzFu*Z|gP**P;$a2E_J` z92@jQ;4MT3=oqj-{HN%Y?|fgW_$SB}15d^Dgy+mWXjxlbzMdPLpIvfi4||zqoc*kQ zkGVk~q5T07Zo6pzXtLJ0>yI>LH=L=gE$&;E@UBzomYRD_-XN&qZLEy z@0*G&`FbBq8{t7wE9N@4eLzM`df3Z=Q6ZP(+O$5I=uF5;w8l2Z)x`!RzD#%;?jKef z77+TMvV*cBsLn4HXUM~RTYRSoe&#s2y#;fa-`%g70Dh)>wDk*z#-nAj95UTwU86P2 zZZmCYZm1WklQqAp;_7&n>#M%5$t<1xw(za9NMCzdbFJdbSMukho4dLns{bhtEzh&0 za1EAqx>WZX*-dF0pD!N}DUO>To)=gY@k8sN^j)pC#N3OUn-rY1DQ;oxzp?tbDWQ3a zc0uhU#)Pl;S->-kTI6-IN5UuY6>`$E93Dr*m~R{@cAj&QTjiK&nQu%pWf=Py9_wyu zmZ-{W!|FRUq&FO`=umR4bZeQhXlc=@DtFWIN`2n8g2T;k&0jW_751r~VeRCqGHx_< z65Nz}_-%zd{fEUYj6D)OEaXwl;*_eCw4{cFq?o;lqmoNo&5W&y*b_A<;;gbq;L@~5Xo3;RcQj(rOJ`3yIPXcILvwA0lG>uc)P z*2UNGDr!n@l}#_+QB+pFra86x&&TZx7qx6R|Ijd?sHWlv=r6g(Q2j|xs{9{8IMZ5w zDCSBlzwoUACn7E;KWX=KN=WS1sGP)mDY31}qZ1h0?F&Hpq;*KDr3UNx;eqBQ*N zvSLNeKDEB0^!1b1|Ea?)J=L0bo|+_U2hV4&P&;-|isF4seeVZlL|luW9=#wkFScLe z`s5p_O-ZfeiXwlHdy#Y__GS3zA(w*B26pjvfsWv(j$iF=;64>jVrm>$t^MswonN{C zaZa^ZjmPvw#yhrWR-Hbed06B0rjn+>riZmLRoSJ3OZ&Z3l&q-;QCHR1lyoetsorEN z)BjoDR9&Un?at+_cZ_joi{s=5`3wK;;kyA#ef0gfPve)j4o!WSGCw{dDk8QqaclhG z$dw_Jf(8Zt=pQN1kXBmNC!xM#Ml}$WQf= z`rAWRL|uq}8lD?@HzBKab<*0_io|URt7E3e)Wsf+{UK&t@E?j$`BXolf4Cn!aJXxn z0sQOYeBM^u3DY)bG2%_-*p}!ymOJ*JZ6l3KH5)Z;b=%dJMoFE#c4qyXs>5&7OD>i^ zXnNQDxTdP)my*=FQF=eiKh2G`UDStcy|^p+eYyYhgz=9n9HE}j!eCy|w#c}I+=Srx zbJ0J?{gu!+x+-EsL{m&@ckyJcjrM{mS!7uGhDidpc8e zUpE|8KeDffAD9Yw3F$AGEn6IPD(qqCl;HC4hUn_3X;H$MALBa28Y0ADKZgDo#tUl| zvQ7Dq?`A=omG#ran73H<+c6lVLh z^1G|Z^H(Zk!|FnY2aXTq28Tujg>ym=DlZ|bfHdeyKsVn-qR)hZk|o07{3-DDa*g@M z?Td&Czq`6Ia?S+IgAUkwI685#^AGW-cbbOU{Ot{vnTD5|7)=E{v@X>gQ<_>qc{iag*u3t<3(7 zU1hy#@8$T*F~@buRqT1qe8K&cdCR=Ssrr}PpZO2qp<#$nAaL{S@Pj%IzLBm82Eg}0 zs_;j62#SV>gGa)l;zKxPui;NZRM=nfyg+zLaF`zsFBNir6yjAk!uNC+=(ujgLggU7 z>2>!zSGx0nE!>u6=YmJxw)TNG-o^TX{a?hwd1@YKDX|Q;Mp=(r|FP9tFI&I0KDGaY z)9K5df$&i|9v&+HhK~6ICzU&#HxWJ~G?-PM!AbG2a56pxzFm64mqZTE^Pdq^@m=r( z;(^b@Zt!x@SMV1+Qtg9ZkX`U-(q0e+zb6lP?clxO45DRPVITMj{t7xEE~n7-*qP|8 zvtO`3wU0sUpZ@lETdw5`%SG!1`w80`cw;GW0le{@b30tAIAQlObX9-ETK(7MaSGs9aR<(; z@571pp3XP+la4`-UmTy{OnZ)_*s%@g-q$;yIj^~%z-Qet_$GBY+am%r5B8X1_?^r1 z9LKrs#fXa63uo&6L04JaznKS!>^73ifo7iz|8G6uImX0{;b!yr@WwG3aelLS8Sswx zAvYX8z7`?J9?Z6$!!y5}a~B$n2JsQsF=stP;5l(OyfyF4 zab9$8f~NqNvp1sOWx7L z$;ETN=8klgco6@PN#*(SMl&NkZQZXq zXMsf}yiUGme&YSi|A;pZu|6j-HT(|zD}Z!6yaP;U>JYDUBD`@&Atu^9&fo3^_f_X$ z{JPJX1rH}TJtgSh+3-Oa?3TM0yT&7bJ4D~*dU&p0ZVvYf=I>)X9h^(-*^Vg37}#yr zJ4U(rpziP8Jl9%RqjSAyEjNqTjS2Or92;CMuI+X_fw!ti!8lJjT6WvD4{=>P!l%kQ zW*s6StoIyt`gkt!{t@itlZQ7SkInrZYz3bRF2Q$MI{!61=(h1}gb&2;9krbAdH3M+ zEr2r;)HTrA)$ztHce1u zKSTufcTa`yNW0^jvlnllI8KN?Y>xGaioM7Flu6_d6nx7c?b_uqTgF&Ucx2#TTf`T5 zPR})agmt(hm76TO0$)xZ!6MHQ$LAIpO}%wUe3P-fcT(Nw!?{#Be(@^Epz?9)E%diYRB0b2I$trh3+Km&1ok9AZfx zaE;?;^G7n7f_Ctg!{Ptq7BVlmPVQ*IVbLi;CM;m}@H6WRZ|{}dBzRK$leZa>8z;D@ za2Igzfmc?-gP#ez%-x>5*1t{nO+$3y=E>GMh%B(q%rV>aeg>;$h53;s3$eX?5nE91 zOz>3j_QUt_d3d$1McnH_qD)z>@0YNrv`9O`*Pf;i)6Pj6)_Wqf;sTK zaZD5?-o{_gnF5a_dt7T7xyxYx%kh_WJNVy0qtc**AFdGNZOuLPnFc|9R(%Y-b&qdO z(=E4X%(q~@YBF7NX*gm*meeD0h)+qr_NnyAm!0%GtBeX&hXg2J`>zf-uY3`_I`~;& zc;KLbEq|gX@FtD({9= zh4u|z?7vQ85-kw_tv(9ez~TsURO1~;^ViMi}WwfJe^a}w=P>-tl7{I*~n@?MlPJ|?j-a(Kk^n+kpO73H}pr5@f?a z2E!Cm)JkDB(`4sZ78$~{;Z1kz*Vc8fDJl(oUGre!z3b2JR-_?z$)E5ZU1F%QJa!G@ z?)90bTo+}IHAg0ej8`~<17bENjE^ggdlG#r+$Z3wxQsiDtK(_J;XdPd%UtoUM)Xw)U1j zFQ;ad{%cKrFRyy2wL>KVXaC%DRQ<8v<{FsRvojcV*RNjtk1^blLTq+L$tsdl{+ zM@OBC`!V@q%Iw$=!nh%S1~L-8ZK7e8?zm~A<%uC$_gI^2{M`D7mlybsL7TBbHVt4=D}@Z$c%zivLexZ>>26P?a{b$?oI zl4f5+QbUG$iL{%4Wx%G;|E10DYRTT($*0Zm)+Mdar6nb=kNG#ae{fd7aPfMxNcE(# zZ_}-&Np;^f{;GPTD=;5)e(4V9{VwkmmK@t7_E21WD_bjGVqkpxnB5WS%2@wY@gcXr z`HWiA5K_7BjVO8E`1#aVzo{DZ+nP7)!+jP6Cr1tn&q?mmb!qSL zoU>hc?YFmb#`nfdUK(^?dPAHjsc_C}Jn~LeZmf>0zE^p$`eZ|xhGS3ll<~qOalrwJ zw_5+3bT)Zm`|BC)+a)FYCHxk-F1Ux^3Q1e;6LS}>wIQLh)9Wn{18=*puDalN^476c zN6U^|u68S4WsEmX)gQI}pv+E~mpCnDea7-WPx`L-@aInR(ws@NS}kwAKeAFDB$&=? zcF)n7Dur)bmr2Syl{FThE*)2!sM~Dc&vEg!vW+2^6R)H`PdVGBE@N@mQymjhFDLvR zpB)*m*dv(i-fnNLf8Eerwze?*MarX@x0YS7oN7At=h?5$@Xk4MAJx@ZEqagU9cO6N zup~p$pwu^AbNWyF$ldL$w#!oT6Nk6Fgm?K^<`ljnPjtZ{s)=50>(!rH=jx}pPb zd(~+)Mb>_v&xDDJtcY2yCZ()yJubC##UO#Ur#cg!qorfV zEsA*?w<%?6r_V%u+~;P-?cnqL%g6>P1?+EuX~TlF3nogp>0M*rm}NldnRgu z&;La2C8xMrU25I3%HJx=O8OPdd*yijU)d-`YrAYXYMkdNm4t?zh`1BAB&n!_BI|Ny z=Z?oS#aXk`ziF+HyAhobGEm0hFSYh>t|}i_e6e7_vlY4HuirQybf(>D;jxTkvh&OH z-!=@bp7b{Iou8*9WL?zOXhVX3#?PHPbuo25o4GXWn=TW&`F0GA-Wo7JC@{FcWWIT# zW|n$kZT{Qu3ttpxm)J@_tp2)waP9UQp01JamP_S51B#-I$@|lb+qm16Wj@b3oDteK zIJGXJZT#cNAjJm$0b7EhcjMn>Ou?wo+5})sXqWZ+3)BDaZyORF!m$whznTwtk zz0viRcaB&T!Hw`q{T~06KEBLcNceZ@nVsiP-TvrxS>eUJXL-L?!3UOf zsQA2iZSZq!5#5ts*Zyhef3wzhs?F-t?nT>W>GRW1#ODNElx&2@;K}Z2Q{3vSL*I$K7IXa;*&*p`Gvc*UvPhR zH5yAi=D^~FzY-S3pGzIoF(s?HvQe)TeH8X%_+1kHiC9)bZj1RB z&5D-iEx$K*uTH2t+_0nRbi-c_Q(HdOZ?uaLmv5s`Cf@1)bJ%B5-J*|1y^J0bqmCUI zy*{EKWK6&W-|^z3{Mnpowpr?S)vnV0CF0@{&yydwSzA%x-{=^ z-dB5}X_vLTSO{Ob>*Pm6BNFZ;#3yb~I+(t)eQ_H}a&AKB)*X^x#7_$TKyF9O0Itu+ z40=tUZ>Tebw_Iwfs&CV@w0V7#OFc_F5)t}5){CAo!q(DzzRwghlt)5FhxtT?Ml1^};xMwZ5o1XpM6sX=* z5&Qa!sveGqoRyv}&gCMR|B>Ld$hlF+5@sfsr9DqlCNe3W4wu@tj-DF4E6f?&HGG2f zydaXhg~xXrt%c?@Eg!f1tbV9gwybGBpoxcP{+0Tjw#Ar<%ypOXcKFzQhx?ricpa1& z)GBZnZMq0~;`g6yluxcWjQ;@k#$OFx)RUSn*Q+Y>ix$25x!^(>uV(k#{ZD&8d|KAV z(5vNf@$#Y`ni3CYsK!{%Ie9Pnc}cu~QcP)Lr?`&MQ)5QQeVmk=wyV|2kWc)DVSmLA z3u44Kc@i;4XmKl?yBu3AYxPSF26K_6(b&)U)SP6U=t$vo=G}oeW?vEGJIpss_C&tc zKUt9|6Zo1HQHouX2fW>|W~|}{INKUVwEU{>qKdT{BF&w z+odx#Kk3D_e#N%hW%i|xWx53Y7;c^T5oWt9d>4m~3Mmbm9~>L*iJBd^Fn&&o?m;s zs9k9rRkU%MYI*tgnlQt58xLzfHrH9$57u#Ql2?j3N{=#Kc`VQ;^kj^Gd_=^@0h5BI zQI8`#1#OkjRHQ3f`@Tc`xh*0MPlbIeBJ6I!USIc4=Y3}>Jlfa8kKj>QE7o%Z1;erO z)BuXTDXNq#7Tm?&90&iH@C4JxvB~ZXUDOW8-G|aQ$8=p7@>dUmM&xVs6y!P`mO?ur=cA zh-*RJ0<`|a0w&30WnatQNXv!inL*IvJ^W_w9k~5Y!-DZO4}R164o1d5 zBJ_u6+L8QWf{A=RELKKNH|C|s;MiszWd6>0+%&|XY}wa1xo%g@_=arlM9tI6^5XDX zyLP_LP<^%H57k4H(h#X$sc+*kGPSVLbrg12?Dl&i707l3`bDe@?-*31>>j#1d?;4$ zE(9(Q+=Ug1`@VamcYSXAZkK$LOJ}-K468jbeSM>T2oDlKBmj z^^;ZpmDZX)Eg6O{G`qCBZA09jxUM@scGWHR}yU@ds*Ze%z@Gk+>~J*O8p zTeMzO%!@_z4h?TSZxUiux42Z!XO=m-Lx|#dN#|;gs{gF&!zx*`+OWA%QuJl9qw!O7 zO7o+VR~4JI3oKi-!_=#+hdC)OjpGp0;;U4&me2Lw?AJ9!5p^-LZOBn&>o8l?vWV?L z=K@a!Ukg6xH{B;t{#udY(}7pZbQLWZPvo6LT$s6tp7#ppWUpd350A5p34j;ZHJpR6 z(R>EGS${;!IEQ)r5Lgc9IOkc1n06qpNrK^P&50&{-Q()x_3Jg0)PZ%Xk`0yTR9$r0 zs^qE(wYeIjshx3w;iz>UJZbIYRPjT6e+}s7UnVc~Ul@8jYF5;S(8D2LhmMQb9)3B* z6g)%eQk2UZ*L zsU!2J=W|bnXOX+wzQ%eNdpFLTEV|y>dUaD1r|G-;{na-_A7{(6l6-*Ezy){?3xkHq4n=-n3}r{A-X1YU3K%@T{T76aq?}$t@7n6 zmo3}jG{l(mJ>&UnJj1v#@`|9N{x<0u`O47T7-Q7(kVSzX2Val;H`)_=A)wiRy)r~8 zP`r{&LjMmCjp7a#d?g(ry3Z+cZuThnYq=`U8|+y~1Bbc6T*khn&Dizzi`(C|-x=(_ z?K+Q$yF)B%to5cZG^Of=TC1j_Ik)a!)rh*YEngTe*Jqc`svM}Uv;3!iUOBFAtWjrI zBVI$E^BNO_J!eBjT1B0rjbE^z*?&uD&&cDETfzP7kiwP?itq4 zEiY^z+Rc_)U6dwS+n|}NPE^fm6xKD=D>d_Vz3NZAOQ@Y=SZhAoe5|TV<3Z@Odg~oi zzWr-XGXI(|QY!a-B2QOL@&7p>FX(0XyojMeWBonKVG+k7W(NM?CxYZqV|C(7pN^7w zA`|R}yTRf7c%j@KoKR0E<~5>tOu#yglvym;%a-)Z|p>oKZss;Zk_wiLAdq#9aZTI;K+z?!vDb-aF>As72M=QuvL$9cqp zTcU^92gLK8ZeX0@9>#^jjNH5OvIU_}!TK*iw$f|`$|91RuxSbd=Vg=P8 zqSvR0<~*3$!8yid1O{FYPJ+jet40*Lk%-rF6T4gC(*a!Zg^Ph@i_Xq^oQI*8ggD*4!%JZ}rGaGoEo7~VKUnXTkjbA5On zcyYX5@MxR`D&2t;x4_ZI!8bAo5qWHgwlEz&sB5uyHWtxn4|B}_kEE*rkK$^(JF}zf zaU%&5B)Gd3cXtUcMT=8f9EwA6cP&mSP+W=!36KCG?z--~Bmc?wKl^O5NjAyM+&fp! zdjzb`iQsi+gS}g!lR64V;2bGXE`=xYWoev5NowhsxK+G_Cx`09RKX@VgjA$zB*U3` zknl}-DEtJbXIEjRP`$!MTU?FjSuqA)!fN;dH;^}={c`2OSOdI9n#~;i=N`mkLQ76Z zm4ZVR>o$=?EPxmLcjQ4h2$GB>>*CcgvLjMK8p6lC8rgyz0Y`HZIRV>%4+!{=1hy&Md~P3!ZTCcjH*^5yu#<>^KGzG|AP1O z4`kF;N9{`#+ev$Hwm&|*Xw_7#3>km9^gD|;MJbM zeA5j_v6zSifAb|Ox)!1hQe#Q7oXEsES%_?8_w+&bR}Um~o`G}6GPGn%u%BY{ ze2!y&?kCgocg(8>)IvfdWcetd?LS1izbO33s z6xx2E^c76l8Ilr=Jfx2+7D?&olXKubUpM7%FVs=IrQ+$S zsCoVezQQW$Dfsovs159Y)No=YIO-{2o~J4*;|>i`WWg7_E!gPs7*S)$%hDu&L-9}C zp?ej_anG-jPl{&YFR*?Kq@Tc{4V4=R9^r>1fJHt+DiddcPivPRNpJiKNGO=)D-#xq zP9(-{7A5%)iN<)k$sdObX*6;w7AaRLQ;=pn1nbaA@?C;e4n<|TGBF(-`f6yyljKb5 zDN+|`)R#laby5?>dCG(A;O_9`Uqk9~hZfo(Kc zvWF!pMW>3!77oZ?kgLd5Qy=B}EIS<@}BL+H!U=b`7qii2hc z-w8St&`94-w@a_kb>Y8IbBO0+J22$0`s(5;uM}rvjHQthovKKnk8$l(Yqg^d*8{dd z>eD=+x#68&Hmo-^)|K%iSR1iiNOUi;jx!s|XO*5S9-n_Ht7&@1x4p@lPo{UD6O%r! zN?A~7wLkS3{FEee8#JAeBy=&*9^0h`6aT8_qiRcHxpd>Q(nfVS7*N zsC~Mn5jj!eNw$Ei_i-zTP%1BH-{;m7>kKgTIpC%W6Ui7&}(#v-%-uC-)BEP@bkiV(h z9sW;TRGnq9UqhRR=fryABCD>crmPYYeKKf^YPrPNdX`l!pKk7Eyji-alr{G;)+p+p zccJi=xu)xu-$tnd)RBtVfmK!27gs-Atx23OZem8G@TXyomq{E3_-(5^OTe8*DT#?B?4BuaCbljnc z1|i&2x z-}8rat$m?oL)q_n`I*bo$sajiTYl;Oxp~r*_kSc^$c!}=c^@kl=!aLnTBCE7H&I(+ z&%|DhY#BT?_-N=qA+rN=HD~ER#CMKCmPw^M^S9s_YPn-Aa#TVt)qJ6q6sXMO zdk0>KI$x!4&4s9ry5b*I?H^ey_!NP7KL`it*l?1!FTs%JJTU+zsLy6O|6wP5OgiZ~SUIkIEuXpM?{rJiW$ubas0 zQBgmv$n@T@H7Lt3>QeH#tiW`}(${rMdPWEEPq`_o(fZV&9w84yMnz=B@^MY7y@_2H zJvORZD5($Rex|Hs7vc%lDD4$RLD86NHr-xs#QaDXA`B<_~_jjEGs3-XeOQ=g^P zo=9t>vgxHC%F2!V%=ev{Vxlsc>B9$Tx9AsOCp zL}z2KSGob!(jES9%{%?1fMt44H%fO;Kh%(?yQ@pon+&g*XP#JIL_HS+ly^AZR4$aYn-*OHOXSJ9J0Q&9keGnN}XHX0oaF0zI0(Z z604C~Lj0@rQ)}oZYze1TRaXV6`l>ppTByG8mymxqMO90ESp7o%vxd{Y)-Kjg(Js`! z)n3wOY7Cmo>ZR(dNT?0ta+ts92x>5C1f#1f+Hxku?#F=1O|iYUB${uUFPdwaQ;hYD zeT@r^ON`@<<;GX0+vf4sNwy946ON;U_0M6RYW*(N@-M zz&hlp>JGGd`2~G4>F~l2a)#m&~n3E+q}p;$Xv!i7Ha{m`~+au-Wu_%1%}y+RE$1 zQY2Z{_uho&ZKos9zS{b;CChxnTxuR|9%HIw{8Zl3IKucG8MYJ5H!Qbo8s{z7DfeF< z(l^=vkMKjRA-fc5AM8&rQ*zA8M=XHtC+A zKV@rMp$DB*uTiZ;{$VWFon65EMpve0k^d^5fy-9x+vc67S`I zGJ-fL`;q;;#pm%na*uJvIpZCz?cugb))CeqtVa5qUK+i|Or+c1F;%uawDh$$vj6Q& zb4_(0^tAE~@*9L_;uEZF>XH+bb?6+XGw0%V@U2wEs%7e`8lz^1_Lz2u_N%tHZjo+* z?z+yY`>fl7#$(3!#&xEpP;lf}lkG#XqiEqC;F<3|;A<%`$Vs0=L?JmjoSA_P z!+8EzzP`$?`T%zKLv0IP58VM>o-RX|sH62m^nLXc^j-CKT_v5U3DK}>8+ceX*!Rfe zeuNqCog6Q*g6!=K4rG#Jk^O+}x^<0pxb>Z-n)xSW{U)IwUos|{-k1+tY-lH&V~Mke z>!!Pncc0JdPZY1o!wKjkl|7JldlKu7&OC#jV^lXsKN*7iXbNo1t6|JEQ)$8>K^o4q*p_%@xuC6XqvrX;9%3uLo7Zp(-DJznsr@{=ZELE=7j-9ri zw)eK$cFH!@QqANmm&;a^%`ZuXLMrnq`aGX^JFSRu6Dn|i?l%8P4r;yi+U z#>QT~f1~g#DlBd?iCVx^=05VbR43I#H50X*E>_XSgc3QAB|m%gOSa;xx7=^8t6=$m*f^# zDh({t8}rRApjLJ}7kb*@nlBcPh`EYu%5*xHt*wgCq-agL46IVx1e6Cv2el3U61+BK zb;xfa%|oJs1A`m^{R4*R-)gsjli5%e#m%OW9w3hL*^}K{ z<$2EOKw8iF{`Saj2Ug$)^99>z&oJ!V8^|k^Y19^_QgI0!bGUw!@983r)(H9yLDd7h z>W}Dt4cH#^E=UZy6E(N;g4h93KSKKIhB3cNI?p`&G3y7%F)({q`F{0wb;np=79Yyl zpWfofgKt+;x}~IiC-XKsf8jfYwu|{M?pb_n-0wl>iQS$R_5o!-=RYr4Wcmc#oL#=R zLNHtdLVz8{kc-%(`bnX?!ghr}tkkATQFKa(KOj8la^L}NZq0CuuF^|cal&VU(_37-V`$-v@b-a6n6RN7~)M`SJhoz&ZvX{b^b32D$3ju)3JT1X?^(PpwtEO?Yz^6h!b-zfo~`@b*zq9~oo ze~Zqk{cqEC9nzaU4u9uaS)8AJ>5G(DkepOT2$PkKxF>2_y_p^h74|WDp>|s2wyJSe zevA25=}}lYze^$%mHA29%G##^9CT@~H5~D$(8+hr6XWeE`zSXvnHnr@b^KnsIZ0nA zIz}}fY1aMtNql|%UlHT0bcs!m_!5#C5TI&BypgBTS9ODeCxvwhZ5^~*yP8%CKV0oS zQ^W#Adu0gH6PGGxxL28i3KKIcrPANOe_fff>a*#S>HE-fcotMy)}V8th5cr)k#)}7-3F-u0xvDqYm(g+0`0n~T5!Yh>iGC8=C}@$sGM|U_-z^oG zz`-v=Mum)h17_5H4SgGwB^vauFAAR>05r% zdJ}UpYEe)t^>54xWCl8@0fx!pv%m6kMCZeep)3lI`5(A zQXdM~6SOX1ye1!s%K0!hdKEM|s$HZ$FjC!$jZ{{V-zc}}e~p?RTPcQ*N)8&q9`e4# zdUdINx-gD;uT2j~2>hn$Ej=(P^My?JkICPvrRHS3%^8(Jf1CGtV%ij2AH$29Wb>sR zPj>P&)MySCv&l`~wtqI@`Q5L@w!2It!>pjoK`RX5>UR7)uCjJO@W;qSk(+`xYcH}7 zs9SPxa);_t$eBtHV>(yr7IIDvm6olxse?7ceNKMBH4b4`sje1G|AOiD^YFM36_I?d~MaCey5tVI}azxh>q+vP>;S4)$-8B?hxK{diR zhNc@js@AZ(X@O19-4E>^el*ymeZy{1u91Hun)7diuSQc<6JsxgKi5GY;i_!u0`_5l6a*T;INv|c`y5td#pYkp3h^fzDUCm;Ph{YRGRd>~QxE7+TrI>pvm zO4;)kC;j(o(X;#SSLerz8S37lizB87byIz!Rh*5tXc`1HjQS_yYCtjTQT!!-^QS4a zs>HAxl{?3cjy)LMi@WICVA~DuR;FjPvZv~cW*vMAej~3sTbMeO?=J3<|2fy1_bTt7 z^ouF)-+K~0UzQvD1iY%-sl&{!WVadh=dyisKE9V<>tCqeU&=YA@EK-@JPYh#c%wdo zWbbsoj-h3^ztZ5y>H6{XBPqr=%>BC%tG*w3CT>s7)^YtKm#7+pKX<9PPx(T}6~$3? z+n|?$H#85FoNuT_Rq|Uwhk~yK&5C*!jVOrEssE$vm*+|T^fR8J5kd8Dw5;F0Wy`D4 zLp>#FLqARU@XLqFsSy^$x$0KwSdB&X3$mzrwu&w~s8{&0@B=|#`P$@gRK!}4SGc<& zjbc|+dlq{xyuY?Tan*I(Tx?2k)Rv8GgeF_NOMgl;jNI?sY}A#d6g|!%vX|tp%%7kA zZ`z!%556`}eOGKDM^!0m9N8wh^{56B!G3FL>awJUv0% zuLI|hZ-y0NRl{-r({|~N^pD_^mh#&pW>tF~mtEyu)N%b##URId%Vujg*JtT5w@A+g z9Mvsg=ZNQ>7tG7bw-+ZA4=k0-tCY6QPt8)Mclp}klP5X2yoQFVxv|lLmh)TLYrp1f z1wr3eCeQy=@^MGbE8jlV8LZL1XoFM}$OZBN#;?@}jR-9d3kkleI;E)Yi}x&$O4Y5S znpOL~=JFb3OtiX@XSKy_8Q?Ix*He1~T8G<%_G{`hJ|V;YbNRT^6D4a(hZ~<^t=*ug zDHz-CPyf6#CDkkVLaeBCtag_sw&vEl3j=D|2j$n#)ugMxdNaqmzfixC%f+i?Bx95x z`3tEHL5fPpqRo}EBLV`a&~rV_?1x<4=#0Q_(Iex21t)cJNGD~8x1;MH#~^o#!lQ8o zoDE3O9%23#wJy<;X1Z@WXx3Z9ET_sl7j?;?Qbv5}^Zs6<`rB<|e{Fiz;q|sPS<>X+ zYBBUHQ)uDbJOe~?JBvH{#}E|)I2m_1Ri!XpE7+2(@X-CkJ5 z6zEDq@+%3IJ4EjeY|f2UY!#OJ|C7(sf$C6wLcmS^Id+gtd1tvsSpCKYrgH3swv-Jm z$jI888vS|sJL9`OpD*Q%_P7I&#XYY-t?||RQ=)e%Pui}OMHGF_T$wY*6z==Wt8}h% z?eNC?4*RRpVTKM7H)90oTK@^AfM~Z>9Q40XTvUe!MF%emO$iAKEM_|p@uEtcsfcF^ z)%SIEgI)#y5-^zGK<<(D`Bu4B*eBWUV@EyPc)H+OCjI@!m+l{0B$j>%OI=wqOBxeW zsRmPjZo^5n$3c?S|8jec${u0GmI=yXv99A zTAq{M&#vdj#6nZ<^vuIwza>5Vl>FtNl&Rli3Z8op>1IX6Rk>7aW&DTA=hOw#Q}@62 z`^9VX`<70$k8+Au-af)P(^KMUD5+bYMXO6R z{k5ZbwQ@W1h)qN<%FfJD+4aG}&4Qk3tMW~mPRh+t+m`$KqH4Lvcf~Q;l3sEsr@{9H zUtWLO@Of==)VB`VPfXjXUI9lUwpZF$?NsF-fqkfXs7UT{kG1SKw{l;1{bjjbe$iCl zTHnTd4lB-czv+k&I-)ep7~DE!W{4%=KaE~@B(OAaq4p(qZI{`X#7wcS985+jmr`a{ zsrd!WjeDA&swM0iWM*$u_5;(atKz8;?@F;uG#)AbJ4c!J+t)_P=FgixH~NyC%9Tv_ zw_PrQI_7Vp z0r(>0{3*_Si^1}{=}~E|!dm$abJt{Q)1IWxO6!xop|rp?oA{mUZfF+zE@X!OvuZQn zn~~&us6!g$jpA8&6wLB6PguP@3aqyGrMmR4%?%bgej`v{_kS<7P`|B&|CfBi*mPEqsx_>B0boiA+`Qrf+jv zbw_ofDpB1KDC2l-L+w(aGlwbDefj_mu5O7g%T;Q5TJY4z$?+-P_Ea?wD@>)tY3o84sDdn$@P2#;e9& zMxuOc*}3vd#`&g8=JVFM_Ed-8DY=$F8z*`f3(3+q#SF3uWuwE{4{S^BDObX^ggaN7 z>bN>YvsZIM^9`b(mp7YHXMo37B0j_j~YQywozp)YAATEI$&JEyZbNjheZa3c) zxb8>Xmo?Ob)N@o1c_)8bby^j#s-j9%4N_;RLRE+0G{*4?E|k5={Ki~{%1lRXP_Bcj zb~PMAj!QGe!9ojvC11AZse6el(zyWMAZ@Jot?hyFY_>#OmYGkR`kQi0Pt0d5V{OYE zCg*eDDY-btkKPHsEI%n36uSvp`4^>P?lOf;EIWg}%-Y%EK%IK=Q~2HdSH2wk5CdLc z=V$U7ekAXJwc&OCE+49Dqq@!4_i5pvu+11q#Zhpa?ZeY$i> z+%F9C5An_P#(5^W${dot)W+MI*n;go9Ous#!t$rNr}?<~U&|A#&t`EXy9}O*p2wd3 z-fg~q{#w|1?~)f18PJ-Zpf59ltciKfRAcY4Rk?*|_5OT2z8=4kU&9x1$G9GR6_uA? z$sglI;4-KAmprM;`AD2)+&lHF1c*0<*vG zxdGK_yffGS#MaGLV4G~eZX0erV(D)A0F3jV<)n3sP3MSp9e0oSWO_!xA;I8}5aPsN zrJ)KvSzp4n~ulI{xBDW>zBqo2eXabr{CfovoQihDck(AHj6P5Ihtta_W(=<6 zD)=5<0p8RBM|Gc{#>e1#b>pk@5$K=m_@D6l5A@fez+*4r8rFbfW&+cUNr7X+c`A$= zp&Ua_BW{DQ`VvkJ2cXoC5Nh~``C57(ptYAcH#jcXf4A3iY;*|rJXdv^nqxclAHTQ-cz+$CK@1$$eC*YE>M)@B1j1PRH$?|W| zR{MkmVU>S{@4PqLbI<*YYoIg1@uz*MeZ1ocYDq_IzuU@fa8I(Q*v~jtJ3qUA_kf28 z_TCCOBXkzOK(`&Im`zL|$10am1L;HHwLYg+OigA!<7OtX7g#epk~_g|=DK18OyC}K z`RJE7xCi)n9*?KoV$KSlt{3O^7$ajkwVHA%2Pm(D4Y!(DrkE=qkn%(kjJp^9P=7;T zytj!b)*ayLWrRj8ph3U?0z;G^|lSThyOy=ttU5``Vp}kzVayed~KjzzW@$lQ)##85gG~0{9iGr<$CtGvs`0b z*PUaWDNemB!MV`U(y`UC!LiffgKA-#E7UE!i%|DE;Z^&bsG%$uV#M~~^vs94_$FNV zmMQPSm#Gi+GnGnh1)Hh`BhoUCwgsbS;+Z>)o-JkGGcVB>HZi|4i+1|g25v@12$TPV|aVTW+=5c6JOzu5=-_bXOL~kAhH^G zs?*3GLN$|8NS!vR8O#HE?fp$xOsQ5tH0CdcS>O3lh0EH9Hu0{2UXG0a40Rp2)l|bAZLI%n+}(gGI+AQWVSGg%vAOPdy@_3 zl>A3NK$XY4c^hV|J{T7gF#G3o@3_CY2Ha)Vz^-5t>8I!?0(?ML5(QxNJp#w9o>*C+ z{C81B+U36K%64{frlB8qcJy#eN9}gIeZEb^h&A1oVf$?NIf|T#t}^#KPp;=z?-Abv z{}sWD>O^yJrGAj7;Dqrr{SWP=*D&4LuWSOR;+OMVQRk|Jxn1PDsh+3?sJp1Qqh9*2 zda62F?Nn`qufWgzVeT5svR#;k@OGF5zsCfq+TX&hrO+Sfui~rYmEAe6PtFMETpUlV zO|&Ffr`THA999y~C)R_|&~LB}M?a2qZgMsEqnL`mO8AwXS#1ZRACK?eR2n3 zh(8!Cl9h_<WnJYVYz!{Em%u>Z?X#{o) zF1OM%RD?ee(?QiwKPK>uzPEZP=Dz{zO*&ifkjT!JMn&!kb?V!*qZJ>;7Fd_$QVw3! zT67=OUpNb9Z57>}>PlrOcM>+i4Hvz)Wp7K8i<%c-FI`#QyR1r)A@6Y3fgeQ5wv;Wo zrQXPZy0Lew_O7aqTB~{LJ!!009&erKUf|ctQxy-SGS4oXU`n)tgTr44GQbT|2wM@j zlV40k$t|fujW^_L)Ww)NF*y-E4X=SZ8{{-{I&+ubs3QVm^`BIinZa~VY88{jO6(X~ zCJ3>q^GtcQqKv});wGi7%dQ&xl&>yM&03OSOXa@jCXdUEb!F(s#vYIDR;71jjxOKR z%{Zc@mgTXti|?V(U+g0c^_+0@v$U}F6YFWahb4vAj9?jALD29SWj!VEM$A7+$cFBEPShB46~RxK!4J-3)~qx1zuchf(UIX`-K`zPv0A;m!H{^I5nXT9TWJB{kH+vS4!Ps?uO?UJcr2*spV{r=+n z??2l82uZ(Pw1do#dQ*d{nh>)-?02@N^{0ZLOEPR0&o$2m-*E2{=W+82Q#fVqU3w>#{W)RS(#K%o^Mm zk+e$NJ1{ZutG=iDxbnPrzV(Z7l5x5z%iIz+BF{=@6+X-Pl+L7H{qZI2Ua`RSjQ4uw)Ibo!aha}*R7436>~8%GVD~) zH0?rqJ~@rKt-S(&5kBl(&}6+q4L%kSR z9Wq!5b&Z00gffpcpEFl8Rxf>*ADJ^I{Xpu@@3qqArq4(#&(IlfbIw?Qjr&y&MUM*I zL4UD)E-ER%>db)iREW30an?GEM^TVJbg2739#xsSDl z?Tz_TnGgD^zBwl{x~6?gzmjz-t9@GDkGlm8rKTZbDmI{ zzzVOwK2UQv2B-szR9KUownT=POmxB3~bGYfpbyv}2C7nK`62s4YMox^`|R(-3Nz#{5zDG=t$ZOWKI9u4>i*Yaw*TVmCz!l9?9)tXWjl@A zEw`Qh6v_Iol}1-;5*8hlX}G7Er5dOm8qg%@Zomh9Wi7`KrQNiR|3jeG^+GP;fTU!Mf(bV&yC6P=QPTG zk?Bnzp0PCRdg-rZn}}bk)QngYV-2iqz8j z8Tva=b*^U@!(XEpMzvkSe;$LQleu=;FC{xlT9vRxY`#0UUe4Iel=Oia=dv7G_3|H> z=98hJ|3)nfs}pd>@R)xitKbebMH(Y6@!xlqT9eC$lzuU$xEqr{X=VjJ2o{4c2KF)} zsh?}N2K*h`Fmyn0E=Kiv@XKz;%wcb+pK2@`1GgV5a0O!k^JBO6fuUvqtEtWU=|Su` zb}YLN3i2``$vxUKzGPmJz9gaKaA9)ctitC7so6heEJ)j%q0U{Bx1q4P@rbl2aCY?J z@YUJ^zN6}2Do(iS?TFfKHUC)G2%F7X8A#ax4?|4W{1Uh(@R@cvFq0p;i+V$V4>i58 z;2zrh*z-T+26Ml2G}bQVs`si+Om}4p6|3R(Eh=y-?Pl#f&Wq52OK|?#OqUVurN2D2 z?SB~kC7sKff_vjG*<8G_pggBmR#N85jQ*Lgb9U!@3$~Pc+$Y%sVgB%^`W=h{yY<=? zuFLXjYPBNXn`~QUN;6+}RTV#x8~JaBR8;(Cu6`}+F~1L-UDV? zXN#+ROv(4c+`M0M&SX!>>YMp7^Utim zDVL6MYZkeR8KPWpk}nXDsN?WBPGH2x}%04+86IH>kzOAWn8t=voh1Ycrww(7-yG5Tz z<@{iO?urn1D?U&L{uKO~Uoq|3eX5@NO92T1Z}ha`y7nPIg5lZ6+*$sq+NuuLJmCId zmcqIGDGPZoaH~z~JN$BP8+)7$;03G_8}rfZ6nX;rQk>;mU6AoWmv9V4Dgm&IpdZ{IzKXeqL! zTRNEwO#{px^LNw2@+T!Di%%3!Ego6)sIW!RvLX?cG`}U*-B)IqjZ}tQSKa`xz+~l2 z%EAm|ZYVb>+ChEPl<+_|w?*j(i>*8RkeL|O9Y0du5He@c*3*bg@2X1XMlt-}=4OSBHX=?|s077`{ zUF|7!Pxq{Khr0qD?ctqX&pO{y!!pqlVtHbEVwABysZnMvJ5v^0CX{uzta0A-b&`5W zzYG0@k6;B&Bdlaqx`eJkUuQxW!&7-Uhbhc5E))!umGnXE!*?-bE3O$>I8jtIby*ok zRiZaC+3+El%xamN%s2K=E`=Kj^kpgc06bKIIfTD3g%;rhS($oCs-WwV;bu=j@%0N7 zfq#$=Xh6rpudW$Xw2@MdSR_P)>pIr2_9y$+_y_B)j-r31ufvhD0N$C=NKW`0n8JE49qifl|HC|9 zutxR~kPiz}oBhm`(0|h=dL!_U4E&F1?`&S+X>!zhnQ*1Ev6Ye z4A_I2t;a=iVZd+OgL~M73+A%0+e~JkpnY=i&ZbNO{gs|ZU!er$FzjqZ?CgVxgHVS> z$!ck#*h{P5Uc+V?0?cV14?raL}&RXm;pV)`ly|x3kb+#Pa z40{L17dTUOaZYhw2fB0KQ^}X*+vUG4tPwXz`=DVuLQr6uuBIQ-RTw9;hE)K|kvSWe z2884q8wWhLHCw=}12R#CA;C_5M!%%>Ob_68?SbixV|wFf8O%pufVY?(Kv43~b2+>- z9q3Cut*5_1Rm#HW_Nj6`JTdPA2e1+6i3sRrM`K5SPPPM0cq&d2e-c}Xe*^KW=ARBn zbdP5>>TCa2xDvX0xK27V9jhHH;jdfIk#ApbABVoz)sX_H))Hq1H0S-?rEbDI);H3h zEBp;kZv%K@Rwwd^?#i`P9IaryOcc1|3SF&zEInN#x%eq3HOmf`3U;lU;n2c{tAWYG$<b6nPqZuOic=nSTJhM2o}~(rl1J@Gti`Tk6M;8RK9fiS*ex4{Kt0UO9(#MyO(CH3F*R*WGPkHr{Q%+z1hE?|!v zsd-cfDi$ibrBFckLywfmm*fTV52%oTLmSpa?@Wj4Zw=I>RS6O2dJOnhcjWv$0va|< zu8qu+`4R~g@ieizm4IEaa0|A zOjX`e9>*Dz$~^J`l4X7Y0xJ`L6EmTv&4&tjJ(RXh;7nW5Z`&fzBoAuJHP9QAxI5PY zQ4AB4pnsSTyxRb!=q`BZ3c#_y`^w?}yTaGS*U{I^XYxKn>z(v2_U`oF^`>~wd%t+| zybf=s_a)F;(su~T_U`^Q{zCtBxZB!*1Z0ZSfi#YUqE^CLGf-4lc&u!Iu6;5_jC#~c z>OZO;dgwOfJ$$9}=&N`n(x-8ri*dI7f$|=s6Hq-!L(gacY}81-L>sTB_QKh=9%V)j z#|`BQ>!2WIJCcgp`(cc?s-63ju!4O#lt!E zh$O@SC7)t6d8{TvN__Ec1WFQRwFA&F=u6xl|_VB!eU zQ$oqcNYSa#>ra3`?;hoTIeQ;HP__;QtSg&%jWXR3SbZfC`r?lcA75hxN!A zj7#mIlK&2k{CsFFyOCPB%3dW-!H4)a^lT&4)O+!m4pp*2aSw`fC9-*L$jgA~w#G4y zlUvDJ*(7=Kcp~ZL8&H(rk={$+p)$XTE3{J@3pd|=Qm8Zrua`@$kYq9gs_q9+f9K(2 zhIAde?juq;uwx__;rI67$|l3_m6l`CzoO+}D6F?YmwOO-NlWFqP`8?)*FTIE$Tc9% zfyh#;NW9~LD)&MwHGy{8fcr23HQ#^W2f7_&P$t$!{otD2kQfL*;pN0HXxH7uY&iBj zgmb}0;s8G0B|bsDehL{l&#>CL3w{1lykib9>iOUbw1iT;;AAjAYCIHpR2-DY;|e>WyYG%`@eM7!5Z6qD zV>ICV=j0o>A_L$O^aic66kL-VxLcx8?brY>h?j7iIe}IQ#{1skCtG9%uGV^aF@9I2 z*oI?VD?d=omixmMc_$H#OuG5t$F#&%cn3#ty)cP)s!ryx#_aU}P^WcjDEdbF*oB}Tiok%KI zNC}GlNOGGcPa>?ii#f6r`rC1NH^!J0#W1O>BAmQ0tWaLXJ-QQ#Q_qNHn9DE0??Xjo zNrU7z;6n_g7iF1k_s7HT5 zTNledX%9JBsHb>Le8h@*Hqw@!!*^^O9IER&zC|BuNRIP+$+mJ8={I7av_*N!vz%TbUBj7;lP`Cy$ZRJp9 z6UBAkKU{(LPdKo-gevkB(AxjQ_!SS|@O~HvKEoU9JUQ0a4KC&{+*v>#vz5KXQ~p`Z zD&JtL%zsxpMcwo#5+yf&`{^gH$8?dXQVb{l5oZ#cMGf7|-+-7+opJ9`E>&zH zr}@;%fzU7J5R{ZdAMgzo{oHZi3#0}{!{fasQAK_wbtH$t_pUyB-TPMI7k81z6i0k} zsSmi;1nP>J#p-x;L|jEUEjteD|nt2jf&dJfR3iiO@%CSPvq zHB&>#)^2d!$uh?)_O-AL*EP#unR+VzAok0byh$v)M&JQ|h#uu76jzzqZa;N|sP2Pv z4ep~fX@PPC)&gdQ(_dftT&fra)(UIrTX3O|r z0B`sH-coWuJ=V8d;SvO9g!h!(mp$!tQ8y(MQ7CN{x>B`-@8ll2pW-xj!IBbVtuh*Q z;TFt%MFx`hzQE6J9l5}LO?jBQ;R@z7#0d8uG6Unp0&==k*Qdn!&IkLALOR6~X@N8X z1ZIxyq~LY2PGfxMH4vC_Eh=NM^add`j_* z*hi{_Y%z@q5HkriMx`-is*p%rAbEc|p(m?FU9W|7xxl-{9bVkZ4(~JYI2HHNd6}*SDXU7^tG@{xfy9z3!toJ z#HGsXa54#^*LinJdWOc%B915!`{B5P6-MD#xS<>(KS~dQyY*9^Bu^mEBAjf4)q7V| zcYjhGh0oY;a07fK%_4il&+8WUh_U1?@)v0edVK|_hL^9wDYBLHQ1Kmnst(8?$i)6& zi@T{5kGgzgFa|ym<>Vr)^O{m?F>?*@H>I7-La4G$;EBaje?ZB7mdKKmF!T0-=h1HQ zG^teNh|S~&l7XrLj(xP^hCc`ng<1Y1ia(VO;VJxT_KG>!>2Jbn(&`UZjDy=sfz(;( z0~ddfVyjpzwZK~NAap@)@KoZkYEemc`ByOn%COnOdbH^8mv%i-x)?Xl=k-H05u|}Pb3gaHxEj1LY;Xdp_G=VbHhTbrRc%>Av zMp02w%IZuGd4SAe`p~uDBUOW13Ur_@5NZb zC5(eZ&TMfi5=)c)%iwG^2aa>u@>7MVXay&mKdGBQY+AF+fjb2OO>P0y;4iKX^Ao+6 zdWYTAYC4X7g{#vpr+(fPo{j=f_#z`Yq!8Nf5|18ZG zr}!rLdJ2OT?a5pC>JO<0aap-fNh7jab2_p`{NMZ;?g|iwSnRoTupeze z-zRFpp9@Bl7^P#8dvOV=5Nhn5rYlB~i{UHrNV+1=hTBq2sk8VAyTMhyFzusS+-bkk zQ@KQT4^xNFobCh6@TxF`#6O=lN z16Of?asfGqEF;dzm&8iqC-D}%wx=~6@mM2a9q>7kd-j>;^@4b2RX6Ah#LVI`*$f0#mL;Iy$xmm!D2vM%)CW0RZ_$}@T_cz|NfxfVB z8-=Nm9F?*%#cAa;B@a97A;lxi1%EG<%1&~pg)ROQzJpk2vKW{zPq0IF5io}4NjGp# zX>ZsvmGJA>1^w#~p&v%V3t%fa4zEfzp^Ke{?0Scrhu$*?d(~*^63B}#7(KM$+GG9P zpFnpwC_B$Z%R&@pSS`L+j0WQ76j>B^0wYB7$8`pn!M8E~SSKub@-hnmZ13?8urmfJB>ta2k40NyzNHysg<&Fh*SpAI3O;J>gsbcb;_j7+>9h1bAIbgpROxoDOJwn>;nVjl7RNvOt-nA9mEA2V&Gn zWt!Z|U6sF5s^pJ?JXR9^gIa9p4)`~~8`%o(h^!BMAugByB-@~vj8jq2VFX|a&!$DanJXAH4&y$^zhD*PLf3F9=&;8@zk97>zm;EVG!K2?q)bQ`{ znpYE8N>gM{CEp8Afy<-->X0L_GuVThE!~cF?Oj!u`L@{AU!)=bBujm5y*5u1f9+ry z)+KG?Zo(q^jnpLnMk#?u4V$8=bQ)&!&$`!pB)(>_H5CZMF}A%1YM(2qAsqqQ9wH3) zH-=A-ZT=T{FNK9-8E|BG1@Z!~gmqYPZ1ekp7IfU(0lL^7iADAuW=?>OgkJOq-}|11 zo-g68p;zD#yzymW1=0doOOIfb*a~P&e+P#IP620VH2fbf6TZgia2Iwbd<0yw0kER> zmwK?$wUfjPi--bV@1kK>4C6)v^~xk)B+Z4Fz&_GJTt0ls{U&M3ZNvDnb8rQ)ysYrB zJB-^YO@wtzBWo&c0vUH#dM8-NpY8q0_c8Rs+uTgqRrw+1V%2ViR2By9U?0YG&jMu_ zS1b{#z*mvLyLq8Lk2R$iRNLfFg7pLQ{WD6m&C6eG{zYe<3<-jmtI;98eVh*W+bQXLO&|G+X*fFnjGvR&3FO3XpkpD7_fxpD~ zI#G5KH1r&H%GuB{UInGn6WC=Ru6kejgZxM7UdceLlKc_!5%vu3B=D(w!;+za4!8)? z;huz6i0_pb$y4Pc;8S*e;Bg>Z*a8$VHP#{afCjV?BlfcxeK!uS2FJbtda_xVCFzBk znl_T=Trse8yGh@{DCGsbr96Xo%ro$-Hw*G?16G@SA-MuuXS<*jw&ZEB<9G!h{Atb) z9P>Bu)${!r=ngci9L%UblD^=63f2&|2Eqcf;Fl8mCV2W|=^@DFebC*T3Jri%A1~@iw6ihX}{eHoKvlo#fIbS;BmH9M!|)=yORc_)Gi~)|Os@JHB{-bD_7S54<}T zz_-o@Xi^DyTInkT-ee_uz9#TFmyNvv7OZj363VdRF6iqg9K=qtuV9tE2g{>YN@J9G z^cnLY{eFXIxlFA1yp2?k0vjs;dsT684*WE_VP!JF5|SiogdWg{Rb}Jh#*Hj@n-F@a zWEsZuaj-KdqBZNl9(E4+^J9Vaz775lM*z8MDsaI!1O~$D*b?@bqZqy4fxH?gI595( zKfkao{(^Llpj?r{+Q2MuVlCKfP4I^F2`o8(!}8Vvz7cp}sU3$6r44d44#>fKU=Qga z#0GZw9sbzB$3P?P1TS=J@n#ro;=|G3bQYR{X9|GI7A`44pST&mN`IEtgjd1-!mU7U zVK%Ip2cffG#Tv~ivJa(h*o>aSetZw|;{oKxZj4Pg3nPN_;aT%6=;D!dEHKLwpa=Jc zE-(qJ+oNGA9EtNaAsKaY3rK>gz`Xk#bADy;s;mY^+Hd|Ge{JDjFd6%quE?gyZ^#Bp zeekw9TWAVR*%~;B`2T_33wxg6&8{Wtr9XU=_Qv|yD&alEZbyxFm7IWe?H>G7P6>`i ziOc*aU~S(Cxf=_Ao}0m6=V|3Gyb!|T0#CR-U|qTgo0dHIZ@>mGG?CD3&ILz7Y6an6 zb1hc_`u!?+ETDicbK36%y={;zk#@xlN&{&huB)VVurun;3o6%L;GG!A zkO|4)bM^B_0gVZX~S9OP;2fHXn{5Bwm=cDZ&pbkdB(r<+QYzy|mErA_= z8Ag6_lCIcWc{;EOlD#SPmn7(#u+wnAqJCzphf;B@xtRS zdCJ`ZTP*qZ9gTU4(Xgy9LJdE_>;XVFP=768lkNgrtPH8thv(Htu-Rup8my|ey>~}` zJK%pB&KF>5?GKy$Ojz%WfykHzyZ1Iw+ZxzeH-QQZVW+=W9q%B}>o(N=3CYoF*>V7H zk0af#XsLHVp;ytjufUv53l4T()WIh~QCSXLJgElOL)Gh}dEyn!I zQoNgknCk;4bunU_4nD4p@|0tgD}yb2D0~^m;j1U8T^S@tJk2U#F0=@=+8mZ^Db}0E zLWbS|$8+H2DAephP~jd(m>Zz)nrQP3P}dPq-yg_9E6jAALp@AEY6s9Fg^-ko(XKr) zCo~z}o}VI&L~_*wYM1GxUA3wM@4(+xH zGpU0hkAU}UqKAk^+|5A$ z7m!Z{=sQ)Cj2V!BP|u4%yEReo%R%2Rj34@8erOCzq(h!wAvd9dY~4TqP0$e2hM{-)`R}LBbO$`N^^MaFq<+5eabe-t@XHWf*GeWv>(l*Sum^6 z4m3jcxABnKy&y?@A;zwFHVV|=1#Q(AbVwD zdZa^Mc zEjC6fjgc<}=BUaLx(sbjxov?Q^g@hj$Y~P(Rp5V9b$P=PS2FVIg@k&IcH0i>{HEcq5!Nod&zp!Vh9?oFVkduVwocXPCGD)Qk%4COfNXxUoTGO!;?1bbC= z9SWe$N>BlDNUiFYPDH6ps1XfX4AV0Bt`dj6n*TWDuM*NR2Xu4^w0RA3CIhMGqx^hz zZFfa^dRDh(Gn7Dq7!;st@=0Bc|JNux)iT9TlERB8PY}Zi(1ziNyA-*+hc|RQ0DZs0 z_f(s&a7`&u2}n-tL0TIjNfzUuWZ)Hq@uEa_W5-3M!mpViEkBX_t0t@Q2l=$xDlfaa+Ye`i*H;wDv_5bi0dhGnqA#PIMGs+$LGlL z4Wx7ndC!1!$*zu<;vgzZLOGjMQ*JFtDw4X1)lZtE-CLlIo1rb^P|q>=kHWiJprv^H zVuS>mil}~PLJKo)H6gde@6=kArCn-RL2yDoTs5o zbkwP)PsUeNyP?QMcy%osKzoYncvQGDAVtbq9RB00&*Xib`carckutU)t z@XO+3vZPjZ?NR-OSLcH`n{uW@NIIyrR8v$(6es->T~e#k69Kiv{^-tM9fFmYzp#8t68on$|Nw zH8((`*FfRv)pULd*SGNY7Q)j(XcWX5EId&e#Y3FT_=TQRX!`wcOzBAD3Szy4^Ns3j zqR%@Bc?G;g97QQ!sQ#riDKwqg73Gt{Q~H$uY*2p=&N-m>*EkdR&@YuzJj5{B{~v-v z=itk1#6WRS2#i?&_l)jY%BH^kL+CaNU~PP|T6fKbGdhJ)6#)UW<+T-~?R zleB6Mt%F|*L%c=*$<;?K#F&ITmM_YKs0Gk3%O&;5N!52GVMzZbS;I<4-;tzYxuRc^ zJ`|p=i7%;SR3ofzsE&xQNc$z}!+48hEQB-37vgVCHLq%LMf^^DOR|W1S^Ac76mb>t zH{(@Z^%cn~`if#t$^g1zUsIe! z3lxS)JSSoyISR+QIEbSejZy6Mj-Ik7M01Sph#IMPtNj0WBn4P6EQX=CM3XEe$q1rq z`jQ=V$L`s0DV`Jk)4zD8U-pc`&^5a&{U5gwr?4{|*$`VGCe z;Gf>IzJuk9<(QsP{^?*nIo-1>Dk&>33&m=J-m@<#9EG5J>QPuNiSDdQR96ILU{ZPYh1K4bkJy-Pj##IXY7)`i4mfk_haM#Uwsua^p=bj5}FuB8M}c zCP~ip5|Zda@d(r|dmzKc+7c7mBUPxRK5zo9Rrg$|!^G zNN&?Vlhh=!8HJMcCz^djnId6dlbV$+y=?VSPL47s7WzX3g z;ygzEjOrO*QcUy(;|4lYjEobA^2MAmu3}uk`Yy3HSUn7n17*_lx)T{DSJ=`ndvS`__@ za-LBS<%(itX;I3IkLZ`tk9cOkV(&Khp+iajOu1gw{Qa~9<~jUAa5#yFC39ew$R_L*i(aZp(HOKFh4Onnebk$&01LQyY7 z&nO&mG<{9)DK+|K5`(yk-BZdeEQJ!Ag7}_|PFNVy^O>ek`4wqZeW@^y-QF!7f((M_qv0jb1 zo5=%q|6kq}xu5A5|LtK#8TUr&ixT9G^r3j!m&EVH`;-prU73Vo@`uI2zGt#ZlvAvS zXA+J24U)rjCTUJGkcB2WDM~r^gd`#BUnn%i#=fFwBrQehPtuh28*FSr5|yP%@reD4 zD22)TN3xjy>6s|kMU92RGg(a9mni+2On;MWRyLMT${S0M>WY=~O`R}F%xa8UMYPLM zYf=kRi!w?O<)o->(o?39iW($qdr=o<(wWf<#YP$lQH>~x=$^?Zl0*NMBy>$w$>a!0 z1JWsoj!B|0NyDTHQMEX(cq1*?6QX#@^dt73J^k;s z_^lY4m5SsANfjnp#49E(#WPtbNcNB{qARjlP)H_8ne~Arj~EB5DLPXwDLwX<$tIG% z6f?<9vdObpnGJ;9y~zc8$7CK$k>!cSC`!OLQcr9X){0bOx)NKL)i;xB>>2elOxCe0 zddet_XpTK&v`3W3Bp~a3IFl{J@$}Cm5M42jr}+vdXGA%|B#kIOt2lvsSEzcOiK$FC@hM9C>iVNteHN$HxNlZ}v-mFzlVz0jAWt5GPb z1(JUhGnJajZ3@BSb5@5WDZ*qt^&FG}X>nv#B6&n|h4ngOAH<{+lVTK7d`qKf){0E3 ziPk4-H?c>fIOv|WFpZ(bd!`{$igZP1vf{J+u&4A5vxTu5V7;AK61roxCO#3VfaEb* zuUQE8Eo(EeH(__wI;`DUdr-}ZSB!Glm`Ch2m@Y4lk!jRYh_)1a7uK_|_9sg+Q3v%? z>^&Q;h}T3*?0b45_GYhfP4Ut38vkT*XCoZ;jAEoz#1T1*g^h<;9P|}?%52%}J7$BX zZ`r}}#nNTz)7MlwmR~Vdx)2Vz10Ni17r`II~j_&9dH-T7vja)L0m& zi}nvTh9K%Axj?kS`aW@F!QRuE$pxY^q86eWdcwY7{ifKjiWJJIlVWAPB$Gj;m(xrP zok?Pmlw!Jp66K%q3De-16@t+)&Bw8O${&@PjyLaEEfAfPRHHB~ z1f3Zd)0J3PqPE5Ak;yfpd72wzwJvH&j1xqDAbuxWr!$3QRL!WHNk7INEG;H0Npgtc zNp`Se>uX5!5yy-4jW|MM<0w%pCmn+6JWShQI>j4JgLDMe1JarLNYVya zFD;H)#NLolW~jDH$|(6}Y7`i`z>B*SJu=$Ga?XpVzsL)hM4c2~c8E`;9F!Dd2e z%tK$k`DbG!iihS#C@uO;{ok)RGeYAbalFN@ssE>EY$ivXtD!rlNlPpFtsRa zS=O#>PKrWMXt9Opj@p5>FSVaI$0fEkeMjThI@RAXieS`2ZAtg+8Bqq&1^rSi?3cyM zp3{05n$KqM-cS;)>R|Lk;b@kP(GrEF8Eg6_Dx)wI2gOfx_9mtJxECo_tOZsVtn5@v zG@515_K-fu^gxowbWgI7dNh(xte>Sj>etGv`%TtQ z(mm^Y*^D>!hO}x!?2oD6rZ@T3Gru`F)2uJe`@Z_mLCB<5$c7hM5KjK8K^%NuUK!)&h(7li(%Lm>kaJH zaZz27-pgVT`w7;|Q?0$}L)aDd3QRYmo`Llm>i2h}O9TX7s;X(Bmw0efE ztD!X#RF+58_Y|IDWCx3d-mnz25hJ}3OF*S$s|eT?<&$cO(GS%k>3H;n&TJh5Tb;lt zmR2&*L7E-4F0~I)0Nqi4O``;+sWJNnwIkE0S&vQJMS2d?zliJTj{0NL?ulzzU;9RP z68V(rS9GQxgnAQZeI(hy`T#Z>Wj!qQ79_2iY+=1DNqv$&O!m-~IEp4oPhrWPNPQfA z#U%S1NiSLkNxCt)#-upOW0HnUnv=XIiA%DdNSuak#HpyP<3rWgSh)~oh)h+eS zRHx$HEXh~0pwLXIcxF~gHk-=E zv}W#qQks65hD^^%ho?L7kKM8Nv}TJ!lio(Q0?|@MSJV$MdphNc^f&rtZ&)}w({pB% zCEGaVTeJXCkHhMP)d^`StWPqb-bCGmenm}&=#Qu^rn)XEJnPGtenj<4IuPs4M4dLAXhF&ech)6-es%Jg2)>rbf!GzO>KGHWqCr<}48H!C$sQIe%ht}+Wk zWpzB%Pm-R&`b(-O>KRGSisL7ev{ajH4E5h}9%~J%O_`;{;vw!fRP!a{6QJg#X;i~w%VGcdIa9nFZBfx$O{j-NwL;PJGA9m!4E+*tl6H6p5-1^Rhz2! z4wPX9_5_4s5~KyLsIQ^cG@;eQq3cuo(pZJoSU(9K#433XG1I!oR#@Ne#X4>HBfxHe z!|=q_AA3a#@#asgE3bvr_hB{i39OS$Lrh;FXRqKN><(yafFvDx>5lc+y{lK+8!-Cl zgcSY&l1~K5n24uiaqULzt3c39a-?l@UX?Md&@ZJ{eHwnki z13&gNV&4qb9Z!M}q%Md{$$5kD_k`UoGVCPz0A7pQA@x{be^DD)x=;eyy#;Ek6#JSMBh?-#s~w({&LJO%C8LE6+*9nx`kZTqJq({? zj72-1?qg?x0pDWxF#JXp1LdKQa0uuCPV9mCK~fWI=MP}N{?cGjvRAl=mL)%DyEscw z5ER^|fK)mYK9ow3n|(rZV{{w;6u<#=wKr70D7oE>#PQf~%xC z@S!p%xCvf2(oz3UuqRE&b%k%LhS(KAyGTaiD{XK9_QqYo?vh*Z71SrtQfh+#qOYU} z0w=kXLJ#ceGX;-vHH94Pjwlprz!S!M@(z9n*E@Jxc*5zCatVBp0M~=->X*pJ3;znW zWmEi5Wx+rrVJmjTjSjXKX35_3Z;>4h?1q<<1mP0udKOn6{4B6pdH^W1Hu&6lpX(p2 z!7Ue>W2e%#;NQZ((j@`8OyZlM_(eDxD3`7D-Bk{D_mVf391B=v?R*EMw}c+t2SP6P z|NI-M;NB88ar1)rxkJKr$tw6>xQ{*l9pIO!A$Bt_7ZS1i<0(8gT@Lh=9OBeMPtF0K zGyihOB<}<|OP2|mT!cSLF;$2N%vMOf?c}fhx1<9k?*=}W#rOwdPuE0vf|pBsNG=AZ zVb6B5e7Em_Vu`neTY;UUA>2s8DLdmEA?pR-EK|9w!gy)9?~wEtVTxp&FjUqXeS{|X zqx7V=xBOi2v_FR{lU;J}l1IYN+*9}-x)xk6U+IgL4fnpKuu58aHY;cP3Z=UOLlhmo zqa^K>t-U;aZe=SIy`hqyB{{OOzV%!rc<~3xN8DfVXji4!;2r^=N&|&(`LsZ0po4m* zZ;k&POqmy?*MvjD5qRjjA=FgeaX$%0YoAmYjq&cCvhJX;0`7h6VA~=;8<-oIF1;w6 zkt_>%xKqKg@~?ts&kXEO@8PLZ4)r$@Lgl?Yzbi7i-M-cG?cSg~%0E;-H_%GbSz5#U zn0s3`2cD%eu%5a<`lF`8Kv`q|QMnWN$fsnrB%yxnC{!moCTo0wCjKRgcC!6dU&}K+ zwKTgW(;Qm91NEFb(g3G*N5;TGSCMvF2TmC?;P){3ah5`{XBJid-abiMUxTVH`p#AH!L#zeJU{Xw-UdR6yhorPcCSAYI(wJ#(e^@R71zmKqJeJ} zxkq>+{HX|q#0dw-r26{FKNfzL4i1LPtxiD}Y51%7kl~fIC%l6__is~eaF@U<)hJIh z^>o*>Kws4ux!m@y!lZ5CX)Z5yb>>U_FC7Vby)Ba0aq+&Jid%v9*oFRF{+>V0yF&5Q z{f%;KV7fBIvln@*sodyYt+2sk)K$es*Hg(2ew?Ru&@0XIE#u-<-|;*%rv%!E##gu>lT>=8yv|t(FSr)hW-iRzlCR_5 zBKfwexxSxNWqB7^3a6ca`1>ilaekjfKE;>pzO1ijIjMgeJL!J$9+fmv|H~D-tlp18 zPP^K2QdMiudbO$QI>)OAaQ`|rs+0Uo`(uAn$aCv6{dZ2gG)exO#9SGv>}p(Ufgd{| zMDwd_u5^uTDpwZxw`5Y(VRfo?i&3aZhvjgWITGcwl^wk@w?f~}zE3t(#W^Ml z$%=Zu%K850`nEPg zG4nWY>aI%6Qe*I(Z=Z6cuS~VyJJJ@XeU}rwH>5oSH{sbxaIM7-?}^^#@OG?seZoEW zcb6wg>ZqrCxA+Ed!>fiEc5|8K?}mIY>tZ!4u5x4K|Dd;h>O0Ae*2=4H3v0CV{VUwx zNUtfoI5QRZBzK(?wO_eX{qMt*Rjx2X`Yh1Iu944E&2;sa1I<#N>O1L`==#fMI2Ay& zAFA78&#Ahii&yz7FFPCYTB+Q7&1+I!4buGmJ1vNho)6@UDb*3KD-=S*dT>M<(1EqexZm_WW#IBD`1E&S1;83 z#3w19%Lawq(!P?WD9;54%X+H+a$R@#foF{`;Pq;o`70pMt}8oXJ^tE}|5=GIw{z}i zPxMbuy*!e&_ObQ8<>jlgZl#i^eIKmO|H!$u@`|~#)NYyKInP~GUXssIZPLYsyQ3b3 z-3Yyslu~bdf-|Z`bXuLs^>P#2$G-^g9N9eK+vwboXZn-C%u^ahD0@o=%0^2wWSym3 zP={+|lNHI_egWRS6l2s9ZI--~IuAJTqg8`+!!+gE)ut)%wk*}&0PgJ`xeq(lTS-oP zUwBS>k2J^YZPjLF zW6d1ZeE4F^Q@m8%QXN+JQ|ytvt$fe4I4oB?OR-0_#`q+p5uc;!ti7b#t?sHdX;&&G z(#IS;T+6}&KH%?py`fb{TqoQ*=VMDdb4yFWnpA$MFgd?>QKN!|**|1%&l;4qIrG~` zbM9_^&?q}tJiDNOrs2^iIW4Qq-r`ct>v6V;vcHv5DKH>3wT-(&vO`ZtHj4YYMtY4k zacME{)tFQJWy*rM1;)i;{o-H5uM1Drw&uGU`x;MaCHyZyRcoynE9)Zf1#GxG@*iZM zDHm%;7}n~y=h>Y!#ZOnKNd)xhxo~S6ZJ{OSMp-%c=+L4ApI+l?8}EAmzuur zuJ;^OuFT_RnDUUO6Zb*TrwI@HBt#1&sOGV^VqeCLjawX_68CXz{Th|Aj`04b z4qsyqdhJ&U?2zd$uXv0FV_&l`5@25U34*L4F-jYGQY z=4@Us;pA;6!uUtK0gZtyEZ1}Uws5)ae+9mP_0;`E{xibtj!p*OVX6r|6p-+Uni9X@>5GaNT&mieI43S3gtSmtR#l z6#HZvczYiq-7T31v`G&z44J0I{W?evsa$iO^iu+~8YlB9$_D>cR6=WFC^?hZKXx+c0V*k@UnR7@=&S(a83U)-kXabClm<1e4RZ24;Q%U@q! ze*Imsrg%<%Ox_{7CHzokd$uJA!|#zNC7f8mActwC1(IC;W`>u3Z3Gxsva% z&DDOawO1o2-$r@st8O@YfyyJ!2ifa{rSTLq2q$1b0 z!?DJZUUk^_dq4&0epm5h=sTKv04?-vFm9s10!aSy1r(Y`mS=B zB0*6@u>+p<74li|hBXU*pfr*Z@DuZvaNpnA_qKPM$LBunR=87KJMDKXr&yg8T=|Kz z$)&4{KPx`oBeVK8nX|VB?A>8zSXhzs*lp{lbO#hK#h~ZuRCGA>X&VRtq=Iinn;KANey;Q%$ z9|98eA3$U6h8FpY-=In0XKDNEy6T!}!+4Wss>TDn9PW``TVS*_2><+D;K}hzZEZt_G2B#TXm2Pqeiphm zw3%szezh(}ze{h@F9q7te02hlE#Cz`RHkw}yqNu<9;F!sf0vcOP9dqAhH{ztM%rfD zUx3e?0>tL6Kv$8eLg%@{ zoI7n_TQ638Q-0PwuWW8fy`rWCrFn1X-FQ9v^{BkWqTJGV%+k{I;yq=9t%Dptx#qfm z^!)B^FF2*`fsrv^(OdnwPHx<6^cj{JDh*kt&qDdo4C6BWcHL@y4a1+hWxN;IRzA&m z{<|_&_?rLieUg1|Re z0WYOf;kDlZ`^uT%yTZ7D177tszLTD%z@GicRRpXTyKRoOsv@%Dsrg>%0AR2$D!h_^ zDQ|aP-~34hV~V?&=a(DI{Y!V4msS4k%&!{e3HPn{Hx3?=-jo}Gvh#^LUpvgu%(&L@ zjlQE{k0~kitSQCd)uriE^g6vvm#Jw2B$$Vq6h2G+O!)~gUY-IunFHcRdmv&@;-~R@ zH7C@qf$gNyL1Jl+0F`=(X0N)PDpxUE@dvzzZUZL46d5l)49nrcV1FSCSO|msSw5q; zz}>|?rs^%%YR4J-K=dBP6^S8{-T~CJ5JRRu)VRg;XGlVb+3-|%Uss^-W>~6w4EjF- z{nNEkoIeCC=)pjKzN0DvCeaY! z%r^%nwHdxD4{#GCzX%C|Ilw8d?F)KJ+*90nt}mP!z@s^2Yi?as@lN?KWz9 z*s5!;pPdnouWX|#8(B`4Un+Z6@=vj}=w4yhqPvA>3g;HLGk2({S7t07Qt^X5*?H8F z4RphX{*J;d?h5>D?w1WwuF`&H2dD$0+B@1Wb?psX4HNXA=nw0j zY7^009n?P4`~VL9T76p?kk?jj0JTpCicTmn^itJnn(f*>y7|!c%5_Tp7Ht7A^{UXn z_2HL6`u?CUP@Yv}$@|Mn;A!j-yv!~OEbyfQ8+`KOZ8z~@SrO;P9TYUz3C z4}TllnY6}dhCYVo$oXo0re30J51ir$(1O;fHR@cT)Ggv?0k?R#c7%4N_O&h_r3=-{ zfKJ#_`xZZmzrnBLYiKS3gKDDkiu@z#L&+n73q15YeF2ZZYJuydv#zs^{m;tv)}N8v z<)x2H-Y-2?vZiQy-o4kq7v487DNiq&T`;+Hl;!ivp4JoAk8G`7Q~du5M}YP9S|V5J z`97M@HKkgusU&2Lsey62F(kwsIy~%r$Y$g3hD^gEL$Y42&(|G9E&iqxw9pm!{<;ae zpZIQ?wfq{O*In1FP=Bkg$N$Q=(g0x>h%dJkgJltt*}`;Tv0v@E2JN}Pp|*9g_O+Ch z?=oK~9bA07U`76W`Q2Xoa`)u-FLD+2D+nvp0P!-W{A_ulZKyj1C^VA;qqvU13C>bq zmO8dU`fpf*4Gi`3AtQ0SQail7)xBTZxN7eLwz*QG;0>8RCW zq;*BxnNQ*0*Lqr|HFy?!_P86{`&N8uUTUsyzEU(M?^>R*piTb9+|WEz zagXASd3W-1ORVLorM}{B)?C*>*A&OcRr`E@1a1X;DF$egG;i@9{fp4Q!}^7AAtOS& zhc`yO4GZaMx^Gx#IBBeJx@h=Zr`1m9S7p~=it@$bNNcy5%{g|?w#r_alh;I+X^uftyz{*VlJ@e?#kJdcdA%co?N!9U}fI> z1^tRQmb5W{V9T~|v-8f8RsXuCyW02%ss`wKXhx~)>5iFB8rvFzA%8^dj%pI!FY;X2 zGQ;nNw2*|*k3#kv)|md#t&mUSK2mD7Rbou-AoU z8td(f+)AzYVqi9K+0F%TD3|a$Q?Hoyu~Wk2#=YV0=z7r~$9|C{i|=o0psj68H_lT< z$PM7t2C5d4w|%_lNnnM1l%j_uRkmHX)^Hb?kT%okxF@lnMP3T?gxK`+b(0J~^Yi6r z;Q_XJU{qyjS$dhiqEE?(xxp9Vnf&8GM*SQ2Znk}pm^JQo|JNh4lAryP*T_3TzFpc& z=F{H`+iWlzK8W0z7^r!)mb}*Hm<6FvBR)&m6L-c`t~;Qgrz-GI_0l@5?T`Wg`1*PG zRDI#86P&F$qMK#1=}Xj;v?C+q;|ik+Li>iVi5ePy-_#Pi^@oO1)%U?uRnHxdD>qjR zDlW=hn7!lKwFj+kJ-AwY>4Wq!muB8Sm^ZMjX5O5anZ;fFjrixr)UaIPLD0c`>%J6eU9}HPe1v2Nl#C@zn+ed@kSR! z+z+1~H7>S!{DzumU~f1ZGc7_FvIMfPzUGd6mw&eF-O8qg$uG`ju6eQj>F$i5uS8w) zpXzmN+_6a)pFPj5d=gw~|D?Fw{##VjM(Jr$DaLxAHffVKJkc9BqJ}i_cw)z>MV!yt z#<5QtE?ZQURd%_gytr@iw2Ga!OmB$N8QCZHigBz?8yC@Zdduw%)HRC|T#3aAV{3Gd z>l}62P)9!5d%^jZ?S$iwV~*MSqRGQPk2gPA^XUBTzptG3uJ1TGZYIwG~c3OTZbmccM zc~m;0sPN^~ClBsUxP3RH!A<+Mlp7!3S)1v#X7R($9AnN1z3NubZB@xx6! z6sM)pKpz|;Y;%{DUCNtNw$~jkjHwz|xzF{CbE#&jzTs!a`qEA}<=zUWHBNq1YiFJ2 zDXS6=g(UL-$PY^H`>VXI0!B~W%Hq(j?eq+c6iS8fhO`*1U4$errBr8)(~V`@q(+YOy~@ zahs1Ys!um<&4a=|+&+COR?mUXh$hm&wTFK=O=iAv z8VPGmXIv$@S(%+*HY)MjjyQsLc)C_i>jta~(Nf9-Wig|)v;{W^JG zl))6C9~=C%@~!d?{yM5mu8VtC)h(g7?^8zuOM-Q|Ez!Qu(l3AV^9%R=cfP!}>~it> zN$FSa&TzhrFK>LdmM?m9f{<`Hx{+Z;_@bx|rs%3wFY7(I`D|3)kLH7R-lLFPjQX%2 z^i|?>unBc<4CCJCF!G(xhEC& z%6AuyD$wM2Fa5_qB;@U6d;KwKB`LL1+9eN+T_64+YFy2qYgWK9aH@8gX1zRIIN(!B zK2}}Vm#gMV-rm zNT+d8O&kw9BO(;MIGvtiJqr{#Q+wtRp;v&_pRUXR!_hy!t9;v)B~$?gEicIuB?k$C_43O(=+{p&+nV=CS@dD_uTude2#Hw?cvGY z>NRh)q~=ojh>AI83y^1*mkqE!3oPWmb#s-g9Y?%1q@(!>5g*qc*{Dmy7qwbNj}B{X zY^42we`Od`<6^DS*n6g}dR~(jSY=mxddXrnht(EUYfYKFr~iyA!CA+$w)9Nl{Ne?9 z17F?BR=nu=H1WZsI}h(Id$g`_l5Ahx^W=;aL)yODd4?{|ZsvrF#EPuar1B;{BiBUo z#GB=f_vZzA%9|J`$H&%rP`_E-=?S**Z;eBBEtMMe+o7-H*4Idi^o11auFHS))_2$O zB?3d%pm+ki->2$_+*tnu`wwPS(S)2?S%DYQypo&+S=moKPr5yBdB5?~hvps{XH@fq z6DgeRN{lt)Z+vH(?mR!LtB3C|_E)l>J~|h&kUh;i7j_u(mMDzcFZ6%?q0pw>zmgF(dx# z8hxV%8ct}=7#2pPMtu^!Bz{KHhhZvJeRx-m^$v6m3+&KM4m|+cxTl^cYvELtO)EKS zu3LWGGOTiKWzc+}u;Z(@o|im2dH>G6yN^?I22`GrsEi>IZ^yk8^O51OMDS|-J6)|R zy*9tEG}zkrhbQQ39ZU}F6%?w)rU5aHRncQEVIh+wSG1~MB31kVryENZ)s+^SFyfQZf|I9Qyib4ljF$Fe4h36 z#k2Hh{a!3D+E`^&y=@p3&c{SV*3f(iTrrD(h%4QysOm>v+Tq84HMb!6EWk^@);uy4 zhK0rSi(i(Ys?j6%MR(G@Ap+ZXDV-u&1YEm%y3K~4^<%X@%?kM`UzMw_v&iwI?SgH;?FeM% zdh@HIir0Z0%ge(re#-hYYhKpc>=!w!N^exXRD|Qp_90Fq<~^`lZ)I?f|EH?%zHZzM z$xUD;E4a_Nf2E%)#_GN{6^8#5xhFa%CMtSBc>RzrrmsU1!%u`Sj^Z${vLNhZZK$fd zd=PxW0wqd$Qq`GvYL=^hkSA~%{v)n2_Gz~6_NK5=T(^EzC_Z%J%`Gs01t7yxw`i6Y>;kTi`0(f-mg0Qioy?IHAs_0tb55?~n9?zSYclWjMYGGF6tdFvX<;*FZ zWZ?s!DaT@t;gzbM$|1=NGz)$Y?3kO9v+^mb9*TKDFFGLmQSq~Kgm$#Pujx$aj)=oi zE2ErIKSdfN7e)jlMnrCmJRMOhA~Mus9HhUh*{&*3%u`NP{-hkQnybEpxw4Bu{*VQH z?i$X%j;|eM*1s*W<$KJY(wQZ`;_%Xt(q9Wt<#os#|9VJv^z#Q#zkN~n^{BF5?nGI> zdZ+Rntn4Q_m4Bu8(?BY4A$j>F)pPYX$}%9FTNO7{rP^N%f0{m!Clw2Am4VgTk}W`uldT4{K${Zn&DQH+8o+}N$FAy&6F;AK+ z-oDz}zC5-xsidH|LFtXsKuJPzO!1bYjC@`0j8}@B)SS|n_j4mkx64pEH@6hi;E`~&e zHV(Ukxvg=bZA>=|+jJLshx$|X7>x8C>aprn)kDPqd4PL}RiQTnc3(H|{;F=yvGzx{ zJ=W}swH2ewKPyi!>s+>^bW{mfa&pzKPk4e*9IQ)2WP)(GyGy*%>-BY)I(EkW;3{CaLjl<11s+ zkN{?Wwiw45+UvK&t};#YNEN9Zt!NG8oo(`C@>8qiwvEQ?PXi2s#E5BWSzg$w$-|}`R0D+PUbF_-u7Frr@m=e zYnu++_A6l$(2O!9*X32pd#dxA?=;QS57px_$K=-5HT+@vG^B0FVw2qXp<%avyl$}W zy#9As!B*%?^!4>e`Ma8Knmy_bnE%+I&H!q6N2OeOSl&wh0JsnxWk%UFAR3lq#clz- z{$CC(_BTeYzU67`ZsR`g8s{A59PC(b|J^pzw$Apla;deQb-(3SxxC^?#lrHW@(blB zD~4NISf#e5j`ptQu63@zTqmlAc)Z@`{?7wDgI4Z_?4^8%B1IXh+OGbL*XW+Tg57y-Jf9k%_*ERfN{M>ZJ1gnPWplPsawW%d$`f3`g^gZ=I=n8ecVO!K_uklU! z49rOGLOmaWtvMGM^>LWVn}b?D3S0RZ%vCh1#%?L{$UI%#L#yh#ra3=!HgcvqHrkE$ zKDL&&LR+ToNoCW@0&4?XOZ!CoNt?vJ$9};+&t7TY*pJ|3Y>FsO7QJURY~=Tlf%IFrxzv{at~K zR2TSbPu+ttcelqC>$>YqayE2+@BGv`*;&i^(wXRb3+LKZn_bIX|GLh(UOIQWR#jc8 z>I#G{&fURnamzh_c+LW)ZnclcI-P$5-vA-Zk2Tzrxi}y?^^-f~n-!h0KA=o>6f-|} zG5?(iOMbfcidKWg{6A~LRl5A)=>XJGy) z$^E6fhkKCccOdE+J>~8iz+Er(NW9O0IQ}7!##{O>_$s_Vd9Qj+{ww}bfxmzyH6idJ zP&LkBJ^F7zk9&u!;%-Ynl&>V2e+bhDbl=Ho)I{8GN*V0o;uXkOo$uYZUm$ z0wa7AHiwk>1*|Fj5~xsDe06}NKGrwW_pPrBp1OQV{&jv`U@A~&ZU;{JkNV5}5@Dcl zKv*d>7upKJz-XY5$gwtkCXgMZ@K=5h*av-q6L18p^m$-etisy;DH1EN2j*f2Kr+?` z{0x+ZJNU25eaaPZ(|~SM6R1LeaCbQ=kj0xz)ly#CQTh_tex0RCU{`I#u~|A7c;Wv_ zFG%;IMB!Kq`5iE`cX2a-=Q4mx;7YJAdKC5t^u(@%-Pj}91?Ufd1JkB2R`M&rH?@H? zEJeQm0v`A{-~+7_-Uh~|R%j~lD8(w_s*s3vBrSnn(hGR&bAXKf0Z=8jpvI!GYQ72f zWmtmsF!48Tm*wa%8Z|#Q>Vmr>q z5!-frcMy9i_Fxwg?WAl6Z}gpT%mos`3gE1q!0{W9qfY^|z>W3se*$~$isXH)M33Ma zV#Rwt>MR22H3gC&LZ$ujt~h}F$&in+!S^H`v7dwo z@<F$Z*~G`7;~{VqZG9Gm-GoRw^j;;!b5Jj za0anua>E24FswEUFX9g0`dZ7fB!yn9ylo&%>IqDhP6IOcPtrL5F-3*< zi8ROee;i!}bQM?E9iMp~#kIJ*1a}B7#oZl>v}lm<;cmsDxKrF+0>!OZkl+w|I&S~& zzutlc2?=>KckVrBpDp$1T}Z^CnHtt;vXmL7c_WRVq_$?B+kK1{Tpl$UNtwT?U;{(v za?MywO@$U^9oY@)ke9?Bg91A~r|GkA5S(?L>S}USrnM6}d%cy_!ZoE4cUC_S^@I=i zapa$HyC8gmxOH>YT~X;JO8 zfw%-Ch`ZYdRkGwxm^``Z#|Hfy-IH^{G}!WV>)J(wc?Qok1n6SEcFzA z&@+Rss2Y;0ZHz7u>4@6t963W@MqGA!q^*ui>TUfEmu#<4qL?F8vQ=JBwz^Ptt$WsJ zCKa>j`@{(?ft^I3S5i+Nu#8qgEyXmS>i z?Kb=gBx#k>zHv*`C}O+)LcL8N){8@-wa83B+Tt?4G0{L8jumyw$U)aab~w*$v;0g` z?u7KLWlA@0l4a;W_IIJp!1 z{jb^AM`~xK=>qrECaB@;UL+g8))!j)Y?{eumbUjH!S|!KSFMUwj-8&m zMfB9aAy>Vxr4WbcBr8a<=C8<5uZl$UK7ws9xJUVo^;}hQgAt4MdQ87ZzMa0(Q#a#;!10SFv#VAon>l`^a8Cy6$62H1F$?bG^0qy)F2s?mQiy|L>|vdZc$ ztTSXiCI=lLrxJe|PAi4@Lf<9#G7)kB;m{kQgF1zJ^@j1^c_6CmkvMzGdQW_&PEc2f zvdE0{6CbJ5#2Ye>9!~tQsv_yMv(cXJP1i%}sb=* zm8vEemD8oOaxVFt_*L8^y^)53;c#85Db5oo$g3nm_GmivGZnzb>Z`9M3R5zx3whCR z-4%+v9=lD{>yx9NdgGlL`~oh`5$~Dk9^<;jR}gYzYAv8!S^`c+5$lhApKE7<{}nh9GmmY^oD}xC3V3ciU$8Os99lN-s7qCiSSS3>WkR!d zhk%BKQMt1GMQ0OmMUHZFz`8BveCSRUfO=#0eg{3$C*~WMCfs+t;fm00?R(TiCX58< zTgWv(OwXh05I3<8-(^yfhgs4dYF*U>>P7jooTkiHBI1{D0`R{RAxFdu{^X8G+eoH( zJlHUJC(ti)IQ(a@o4*Qrr=FQNGS+0oWu||9^J&Vbd7qZ2Kh}DA+Q#*YUK#&)uB);2 zy+eUMEzJ2dzKQP+GhXW|I>mX?3T2&9iLQsb@iD4QmVe|tl5j7+&bZu^MG%;&?X~EOI9ao9gP&s>z^QjQlyvBb?9?cHHx*<%;Ls$JxUS87op0b%b^b; zcVt~;mi$C1s*aHJfy2~G93QD0%*^s--wcilr$%;*ufvN1ebM=r{9gOhu+MFN#)`jk zH)G!Cx}W=2t}F4Yqi?xO3Im;uy!%{bn3DD+HE(2puxBI()J%diC7O)aqc;NQ`t0cB zIO5#mt_sv^vGXTWkRT0Hy{%O@dsuIYRO$pBU|ur{eGGZr3xw~ED~@`?7v>DLo1Ba! z&{X0b&GX&(PFxu#Clf_qBa0y!a~Sg&U-5%_LjH~H%e?j!au?l@t;^+uN-Bw}4m@lm zR&Gh;Id9auD$hhQd@P1e48ZO_Q%gz6Nq+pUEy0934I#9v`_O zPgL*A+rm48{elw%X4a6*l+0JzVJS#I@KlM~8#5wDp&U)4ILAn43G+L*mo1AOV?65G z@@8*?ghKMTBg0oGwn$9Z=s$f!J?Z>odJ0kS~n$5-K zX8SRfkm9-ln$khGNxUU5Ab)rcvP=Pqpc);Dbk{%F2Y8)k-*5}L`s@UH9a5ZmIz+mW z;n$3OMXhD_v&*?Ie4@~t&!XECPBT&SD{;ywIY!Q_v{o9)HKa3=0Ujo&ebC1vo$;pm z(Dxr7@noMu&D;-N+$x;-oA8XC#Z%vbI*dB@e^y)5 zr!P?ts`ItY+A#1;c51V<7aD*{ZJ`#_e0m*Yk#S6Ks~h?|T~$jdU z-<8Vfpj#-3at?WroF5;dRhOn}r}b6ZAZ5F_BfJ>MEiJB9-x7)ZELX&{Bg!4+b!T!K zRw9Y&ccFcqyhm%yK(-0jfzw%zWBKdcQEmdRO{_!X^KcR~lPL}U)nW7-PO=bk8C^Jg z7-}{dLejiN-J?q~^}v?y46VpjWVO1HFgTD*LBir&Xl5^;+w`GjZw z9kd2E*_Et7c0&jA3$kE+bQUs|JA-i(#cZUqh(GN%RypkN&-Gb)NASjf)${6~wXn8B zw{RWej19(4b3amNIny*^jIX$Id9)PuZ{>rWDmPYm<%V2OsiKruduWaIYWj7J(R1r{ zwLg>-@-n$7R#kbKk$cO-6d&eDl`)yg&2|Pp(+)h{F^=B+VWtbbZI1qx1tu~te!@7qw?d~ zs>D=45B>#D%`vJ3)sQ?6J$_T@^lza8%Y>TnD*cJxz#L>^=sRRL^t4Tos(c)2tJ~;- zv`9_DcWBFOppQ}n)eK49VWfC3K?mN+6wD9cP9p8$H~diNbx>~Yd@S-u*Q0jG3Xl`p{IUG>_E2iSgb&cu0pRu zGU!BfV+@idms;mcFHWjj#xnh})>nI=c2gy#vHD0o1=M96W?>J3a%@6!b}#J$o}uGF zlUk}H)r!#cr67s?jn+=vu0F%{xT*RyMjNGV(^hE{fb{ItDAcY;@yvF^?h@C>;XDyYU}26|vWk%msG6LS7K0vY%V*`CMHRdxq%@(@@+065)M zsy+RV4nsZlg1$zNrW;`&`wdr7CnMxr@(^Ax$nQ9n&LKau0Cuuk@Owx`F69X1@lF8Z zA)1fOMMzlHjr};4kR@(3Mml`}A@~A+sB@dJW`#9tMig39tRA)K8;! zeMQy7z3T$JE{(3q><70kH@luq;C6G{xEb6eT%Us6Gj;(maT6!;X=Vj8gek{3Q7KIV z=3I*|N`IyP!R%xu-gP4?gsIMd?=uan6X!R6L17L*ieW%eei9}SwL;`H{F~F{HC#Ox zu%+#ki*AK`)RQiZ=QxwP4os;Op5xNMo>G7*6$QpJ5#AQ(fn>e023xv04cF>#V=8u@ z70AQ>s%2|WwFcT|wY6Fd2%xS=FvSTeq}mYcshye|8SG=!wZJJqt2wlSS}CO1=h6tx z0~X##?AGPA4q6hv#~i(&F&ob?3mkJ6@@B`P0~mrN)q6-{zd%*NJsyPj-MHR+xPhD#S8689vS*pq%mA!ECvzKB)&#me*40s5p&9?bG6Kj! z|1YE6Mha*t%-PbAvw9l5y4yIN2_)!F#$V0_J2JeRwX+1}cyX}&dc zSV>q%S?~`jib|<6<-^MFhcxtPXcBC;3-I20{3QM|ALO(6Fz``@Ux7amX0i0SyR`_mmBu@UeIHSJfqU8mHK9vII34*LX8>(FvS!FR&(`QlF_+SV?cmJ2*F=!5`!y z5@zdQf9hcUH1CmT^cLm{t%X#69p9e6%@yF*L%Cq%)K5gEvzV?yAEIKZ zXGp$2LoUUxR2?g)CCMQlI|G=b4)pOE+)mD7`jv$>lS16UuN3T5bMS;;!^d|ZdvyqW zIR48o-h`(qoJr#zZr0Eg|BlIrLd(DVxYG#cG)g-r!uJi6d2bA~bd!bX)-2 ze~WO7uZQb$h9%kI%mnP?Bqka@`ZHCL3LurZHr|a$Uc^4q1G`BgRS>J+KB>b^?`*XL}RSb`oPezipKwg6% z4~2f~vED<=4L!s)%n@Fxw%SWuj&(6n*(Se}?c98`2p0qXcJp50%WcWz% zUhr{PiZl*43;O-l{AQLn3-Q1H-O^C2Jn$kRytR>oBJ9@_kqUgFLz#7 zI`oN+Ml`<2+w{-J+i+wgB4qH~6;ocn-t zwXl+Vg}Fs{^ilJHZ?q<#+U2blW;C2h1|ko>K2nK0B9XmKW&W{p@1bZ{G6Wf8C2+tz2haxji9whI;_?3}u9Fw2pmp9V(6;L?W&<-Dv)=lQga239AQ;%QlAs#jL|>;ql2fQwbT+z$Zb*F& zTivX()&yYN)1ZO74^*(4)gM>GZS2&3s{g8|p*-0y{uCccE9B{NinL5_f{*S|zR9hX z3d$>Kr#M{nMxw&uP|wJ>@P$zMuvdg4U%n=HRv+t0MuPqTQ>+|X53?6pmP>LxbQSaT z@+@@v`Qbu4XDYNLgtMr_#qR+&-Pl}f*0a}>#pxjJWF*XfW3gwZ(Z6&1__cUu1g;!2 z6z6eOstC@GE$D{7lmDVRdv49MPT6Ns+4Mp8c#5vd%wcGB20xinsPK-nf3YEUFnblA z1RJRX^d7w0L&dxjY~M%NLknRan@b1&n~IZ_h z%w1)=(i#79q&iMM7D))@3f&7m5dDg+Pqv$ZgP_?%>;hDNraAK~-uFOq72&ZS8oWId zwZ%ib9%de}sW;}o40ou!4xd_Q3mlwKD92lExX zMI-7vxsqzfj^#^2$A1mZQTxq5;ruiR)4wsSkE6I%xE7DtRl-ciQRh%+tmAj6BQ@qM z)1AG~e}azhHM5r*KrF^w~(Gy^_l$1=5(Yw z`*_xZAN?oq5Pk})xY_JD{*hx0-xDaxFYGaTE7OCsxQ$@hTx3?^-jAb8p{k+TZge3s zh>6xdswmrpO+inP!phW5N&o|M(&#;s zFpn8#&o_elX`?Z!l(yzUvyL$z2tt0~5v%0iWed;94a~*2i%aCD(D?2|ty@}K0=;ig z+k{Ft7pj@5dQL-C$0^s;T-rnZiZKD*!5$;pOxJT`?(ZaK*saYY>RWuizIsmmSGla) z`Wn5IIo5cgH#5BiNmjtlz7E|>E+G8(>~6Hle1-zk$-QBVu{YV$+!^i=Dwh`QC}1h$ zn0%N;AK@gSH2*IcD+PeI9N|U_T^(&5g@otOk~ZLzxVC&U+ky+=`qf7VEzwngUc}-& z)$Cs6Qq*m^p)|d2InX!Fx4PMIL?QdyKh4wn1Z}2qSZ}UVmQU}X9oENdowRl)VJ$LM zbCp&Y9wEO0Vfk#PSg(!R+C8;|+R4<66eAiqF=M6c1Fe(hH>(X5Lv%Bbf!`CO-_vKB zeYI?Jl~obg#$ENRRu9ab+2#u?+Z=BiS`MwgdDfmyekLo!F{dke0#mq|WDY8t-Aoo` zPm)`#!jy}jN1eg>un@||L-=Vo69uUe!F8JgAvp zV{*QZ3|qZ1`zA2Q$YV9vzXDA^YIQI>o9&5IaLwvrcbW%mZ8`S7n^dNGjCg9%e)2Ya$1CJ!_(IL6PZBtL&3?>vbrPLMPEt2h8nM?rWbUI9jXUIa zTha5<3AP&*F;qd6gE);@1xHoEZoa`7gf44=S<84u-6zgjBvTW2daapmeZuU+vOZgr zs2O^kwE-S#V~OW>G`E`1xuIqeOlJRJb7`H8bmweQ=B7Z0*OvZh z<|79YV=;XZRgFU5(mOJ&xff4F4KM@lQ5Av7HK9w8=ZzTV ztWnKQwDMWg*d59l_JloMKSNis{dyW31>exlqzT`$^TrOcHIo;UtYq@EnS@zl#O%u@ z8Yxx*wx}6zj%4;x0}Pd{!PU|0(VuJ`ER|Z!cq1q3tao;9!!{F{WBNI21gfQ#R!`!Z zI*_kRF4s)9q0zz|M8DEHGyfx7t3T`jA;Qfl#FQh>8VktfQ~}M&*r={6nlY#()|-<3 zf^RNoQwONY%2jyIU9?GK0y6|_xvYJL+5;@wkEg#P+fAPiJTe#h|D4QyJ%!@or1fV_QgD72XRWD#id(a&EM%}ByE1MMv^<|Khz8C zXuB`&$Q+^zg(`*(+bQM*%;UQ zpX9z=8@r#`nY5_)W-|TLJfyF3=8<;LL%BAgWBgt668eVT{8X@OUh-4r9@Hf!x6#+y zPioY!+HkT5yV%-p7J`RgTc*6;4Sh$5XpXy}&@Sx`8!#K$H>vJ$;OuA@ASYQr$vk#h z>WZC@yT<36#A(G4_H=3Z`qQkQcQSB!k@I3t>TfGeGW6EsLo z(@3Ta+0&j*C#f$SJ+wV`EH#L@3)mz`HC6=f3bR=Ho1V@3EJ@kq9wC>a+zcuZraI>3 zPF(TPW&uYYwY<5&k*03OM6!q3oykyw)PIh+`HB6e!!E|%lauMM@V{(9CYZ||yMrFr zR4ARHUM6!$; zLqDQsfWed3{z=v+^BP0Jn^?@Pi`3v&QMa|p#$EOS)khuA9yPnzpO|6VT{hV~3Kl{d zna4^pQmKmMd!m}Qox1>B5_kc9&swfEW_?tCB|pHT2V<_7NIa6 z?FNyXIV~?Es&dQpsw!Bxrc-;y2>+oIbSI*l?quVQZd7Mtly%7LLK*rv`WU!fh1k5x z2%4ap62;|4&dEA$Oh#Wk0i2Y{z`e)Oy@1CxrE?lP=mOSQV*s-syUaP$568s|bS%#p{81akS->9>pxOgH@-xFGT7J}uh$ zmspT!!}T+t)7!Pp+ykwP^+|Z6=C!}FF<`cLGOFn|`GTu}bH0$^Gn<&%RAW1po}ulg z*3i9FA5F5Aai#ZJbM(S|ym8$;N5=4N#DY|D0T?NE2AOpl*NpoNhu&|L-G{76<}-)c zzp(j8H@GM-nK6=wCb-(#8f&(&SRRAti(t}phtOAhM4sc;$hj!VUPG_9+JZl|(z?v$ z(vN93+#R%mR#8)D59_8mko&B~@cAs4HlC&E7uHMol17-JY7^>`QPJU-w~|BI`%02< zRZe2Aa@i6~r=mJPps~a<*YrR>Fr~MJcySjdz?)_^gH6TqbO< z24>q^^JB(y-OMZRb@sK>)#GdmQ&AnQmv+6B4^b<*ccB%0Rk)fi)h=>*!8M2@rdTG} zZ#=V=IBeXvL+k)zIx*cS#pWi;kOZBqCDLxguzuxI^&eJ8V=6I=`zSsp>N6wsbz~eh zQawjLvc?nlIMTRkzX8KCol4avqC5RZrudR}#Q4YF0*}K)dJFtO-;+PA6?8eX6SKvR zvA1)LvhrjcPLV`tk(e^#fU3+YJztBTxo;&ylt-`)O+Q}h*CO)A5brrRoq zs4h^s+lB_zdk((OF?$&euUn$4IT<)BOWra%lO8&^SqyxRV^mSFwyxU0ku&H6`X{*W zt~S?DtCZI4d@5G_1&ocA)^cjNIa*ivRn}$c9Gd~IUR#`tirw09*(d168xNWO)?s}w7+34bLQF648ov-@ zFhgaiMP!!YWPETYdrWKuSEMJgA2|0TvmWu;?2by$u-w4EHW5>_mSi$@1QpRW?1EYJ zcJlyq27iOCunpD1YB=$(2b1kDBCoueT)~q>6D`$Bq&w<)?2BAcvO0KY@5$E0LG0Mo z*&EDJZ3BE*ADM5BUzh^iEIpOlPFr?5YH7`Gz;`8M)$icrO~qcn&2B?)M*Ufu7s&0x~5BTjU8^Ln4782aA*1qXWxIp_3dr{sk>0a*X7Du zOU+T%P~bjAP$@`y0dgnVf%-}eH4}ku6@pjsb#fM_#VR?#{IBD20TW&UjDMS+1E)(d zvy;_~+Ch&%@B3LR0)E~R@UjYFKmH$(pQBn5m~IQ0M8X6Xl&)7d9s?6}5_iaV_BC>_ zc}!0r8#1f7J4AgWhkX-#nR7UgTNB0Y-PR60V4blh(1-XYn7zu>6d=QMExTtVI@Te6A?bADwJaMwa=2Wg>HTCS#j zSAR&e)yLW{{f%lXlh9A^#{}ts)IYQ~(m*aH)t2^Xv*-#!Jzi(NFhd=md7(j3 z5`a^2RwD5qQaDvu?IEL!Mep!D5ei}|UQ3wGj-abjsC-Np=KDWY25il`)H6)@S^)!v zOCw(vZjG7{;L>Ogv*X@$S-K_L1Kgmm&UVhajxKy#pwn}hD@<#KBe$Es7=87*y2oNM zoj!y~Sbu5(anWjNUe*t)W5h$DR(>w)bq1G}!(Sw{T&k&-(7q_Ys2NaGzKm3nT4|TH z;_|Ctfvn!?-+%thbU;<~Df966Q=dzGy!B~tRy8hPVY2+I;_LEl&D|#Fkfsr(OY-J-ZFN74dK>dGdW81|{|9+hFQ-cK0r{*l z-6(Hww4RtNtpqXz4CJ!2yF0~O)AN^O756t?n7RuF>_%XDO||Qu=d(DPMGA3}z`f_tD7_CvvarP^BA6@@UbY$O^PU7Iu z@L;RZ%8(Mi6*(hK0AqfIQdw(j#$u*LSbLSz;n)5G(6hQT1DWUjtwNiF8-rG;W+YGO zVs@$Q{{B9}9-&NetUV@LC|w0j%7$d)yhe64|PJAD@lj+KU=G9*?+(ZmZQ_MJ5w}SIZ{Zr&< zj)PFEbz+Z&Tcw{2q$;B{cqN;KfZbFCKk8q{V9)9pB3G?k-=q6FtMJ#Dd)8w4Mx?E> z9VlSXN(54Q1?Ns8v>QFBN&FmFdskyY#pHY_dlTK%RDFlO)o5m3F=MSYI4$mxorx0W zE~BZPKs#9{H`DRayES^Hx1oEnlNG8ktHFnRN>9c68H$N`wmC;{ZC=MZIL+k3EdD%m z(p(`Yi?1Vf!c4fCv_>ljMCXvHpw1bp){&cuqa)44khmq>A$T(IDtIAqAzRIs0(rwm zc(=G5+Ly3@TDBwmNcQbeVf$J1$U-%Xtt=caz~oWfzbJcu&igUf?-o0$<*A<8sYzTGKZ^g%{z|{FTUn*8cw#X1 zgxSmO;se}IraZ;i#oD4b;|HqVz@OJ{}qkkxjz0n8P*2*#Mq>XZRzSoegv(J9A)e zve>y*xWYb%6MQ>!nBGX=Yy_>d#007W^8m_~MNAEPBejF950?B?bO>uONj`xY%4oU> zy#P4@Kfp*BW#uGf>Q}BloP&b!gxU)Z=vHP2I3E>&TI{73fsHr}j*iFKD)d2uG)t*F z5sh*|`d!|P6Z%iBgtkJNB=y2`UoG4sR3P**)IEGZd;<+`o6y5RBmb4GZW)0eH@?sO zStR>@xQBM1=)%`?b#}LM-Guw-cy=mq%_3GQ>pOZEp1y}^*97c9GhFkW7ll9Iu{F{0 z-O&f`ei^PRaD(bCc(~$B02;kSn+JB{!--V}r*|DN;QoRpq6=7p8PF1_WYPb}@J+Te zdjoy-7|aQ8g6DA$FACb6EkJrV;Q1NPYRnFXg>z#7euQV(dR$xZZAOyC?O%-Qs!L9g zBBCf>m!5;+G+7=Y9TK}@$1V^c{7bXbvr@8J`4~1PICR&{^UCKpwW)k^7on_z|Hpx{BcW^yso<$w4oM+%0Hp}_V zdDA)Dk;r#qXTsYgj`pA)u0pqD{FuUgqf)>CXo-`nCB9P?R4K36?=1RTrXZb{Mmx{@@`P;P19XjrJSW3Ch>*sQ(;D%czEKZvnJyO~IT#uO!P~rD>8=x+Fdm zIq9xwMn*=eMv8>*1#7~gGQU4pc0#t19SRJHOjNS;o4{%P!1Y^VK3@fVX_Lq#Ur-i( z9=k_0W=UPRQyc?^W;vmcqqcLO)8VS*%7jDXAK)CX5Z(((j&qKAj#%*Z{^mD=Y1|xc zdJO#p6Z}8H+4%!5P%K*pI;z~vIQk;>6&n5&O!^mq+t`x4glcjmaM10z#}4}xI7h88 zbN>_5>WAR$ozs%k&B`J9v-CzRCa#Da49lSjp~0a?L2ob=2n1FHiU(5tbNyZX%>%{5 zpJbn&u5UM4=zQEr3Ry|+z>|KG{74OePO&Sz5QES{Z|8sIn}BCEi*E_9#_y;`98lgB zfm7o~M<2&ep}XJ~n!=TAqR?C@2Vd^;P~nY4wbY%ti|_IbetzqqK{$eyH<(nQlA2ER z1~X;=u@y{^m*`TG%xh4`U)L^z*K=D(&xwh~B)x>zTJ4DGVSRCaWL%_rq+Ylk-pA!o z8@Stz3uO7XL3QfS{v2E<9#ee!JoBDa5B#EcwnVfb*JJW?0o?f)z^k8tMK=vxu?NsB zZ|5FDBU{&T-7y|+nT?$N98Dbep&L$d?r<5dUARj(vAgtg6on%18+-*{VUL-`zk);G z7FL(eLGZnOVrBv*~iL1BoT^E zLA-zpw|w~5P+D+N@Jpa&plg5()bz_)GqcuaUC&DMSBc!lEY#9|GYxA4bVxU;i}V8K zAlrs(z%}7AxgB6kIEBfMtIp%_2YlvO>3r(U19jvwcU$+Lu5Hj2c81sB5O-^MnEnR$ z&-L&biwF_$3Gxa}_-Uw8#~#jtf5u6%1*C(QxJ%9!T?l&d!q^$#Mqq5xrdt+VfRXMS?xoPC z9`TIyyz<<~yJ+s-2cNb~htt^uZnDFLw!&gQ5B~@_&Tufofk0rM`2+lou|NTmtgYrH za1ncAk~9HK%p|BFXDbge0qd=`R%*$=f{Q;zOvCEgABhY9=O*$t*eR49Ivq3vzF^}( zd>|>n273CJ1j>hbbX#AwOU6Gy$_f(mk@8Uw3dVu#6L>Qk{5U8}JMycAC2)X#?DD!l zxSG2MW0v@@=Qq#)@RB@lJr6v~F|{7+KIS^^tnb|6sEFD8XiU|Ya!KHlod;WWAKe0{ zCiE>(>3uZUo1KjLnhu^;b#S=e$rGSPJt!>zqWh>*aozD$^B#eYHUY}z6z>{ep{RV(O4Q`2RNoZuO3y|2K6qzNbadoz z@fx3>FUt0%htW$ZXw|?DSphbsXy(NXcCdCqStZ?+%1UFz(_(FD6g(TAh%F)+kxdaF z=GHSpb%VzP;gAD=ZW0UxHiqON8-5yk6PyxC4iySShu%gk@C1&ldyGHK2xj_~=!?)r z|HeLYY{zPOCj26-b{vL_-yk>R&F_2Up*`b#ZKFDPPs4qApzoHit>?bynmfgrKEw&U3{O1T0 zsSTdalJL=BR9Fz7hDQfOk=Nn$kQA;G9u&zZHV+RJCy95UJgFj`kg~-Bkz}cxngm|v zdt;^bgcyX5{yr1WHE{^o8RH$h_?kkT5OBci3H zcZ$>$2TL2ln$3vpjhqOTi`0*})}`U!9}_Lz zU6WJ`$~n1*DQgKhkNz;0i~F?wieKueHbB^Lj3RQ7?^`!X=}I)xhac-ZLGbwwjK~+hIG29B#Vk?bao806X1CLK@Rr zOi~^TGs(NA&Ydy8n?CNdRo~2T(P~|?34L2G?>tN_HETLXs`;$D?lqxXe0`$;Rm8ev zlycTru7J_85QxP{PdHSVn@{!7zA$@{Ox2yK0)6Niv!pTA9i^nvvz0k?F_KnJGWnT@ z$}+n;=W5{!b z{JKbg0ZLzlW|eWoH~y_soGD>Rc2njuoP|EKQ5WM-(8z%^v9YY|}vX4p^l45$rf zV~%-~>Y#3P%#8fwOx7k*@kV#Y3o>`Gg0HEPtY&)3%7yuA)<|ZEG26W092;)yW}p%5 zPyepRgN<2$tZc4k&MTL#qih)XNiXml=QF>VrHt;(BPyRXow)9pBGx4%>>)GFd_m7N z-&n<{kK8%ws;hfgVd5PXr30?Dk3mVcW97n}X%zU4A)IvM~mD&(FOio1ACR8hs`O>RkG->Y-{B{Q`bD`2uoQ+L|78xnJ&&2k&j%! zJg}tQMQ&|NpJ2p#c%X<;Pq>~S>YJ|nqHlK zt`$Tg*=VA@orgQ8Vv^}u89GF#IsB2yLYlIUNM`?)Pto(Z9pO~|0iJy~vY=`@_ELXD zM!GMCe!BCBOL;=h?|Kq385Rn}a%@Yr8#o4kDd&Bgvno3`v&m9t@G{0LTZC@v4!Wu_ zn#Mc{tfU3Z6s??K=sQT$Y-XCwExT5vlIsywEVw0xlJ>~+qr~iNCz#n%K`vX(L#-t? z(W=!!dn+^m7OliM3^h^4L-s7z)Vin61joYOy!N0=n6|w%o_Q(m?r6#@48hW(!;w!jyD$xGxYXA z#5=Q};THUsoo!BJ1{klwB`-kIk)rM`^rpa~C|fQqpYaWm%MmVKuxPb}M+hAkhA>Ye zi-bSrMy@LU#jXU*=%CY7mb$zuZNze8$c5Tjq9XM_O=Mr{f7;`X;hrIZ@8o(pR~y^s>+b*@{`8P}togc(tU zfs`RXV4*!ST4;;umf{`HadW)%!Mi8q1OH-D`fr?o=hq zsO4&Hlu~cFY8h2R1>MJ}vXTY#rK?(p7z&oHUykx@7d2aTWMh_jhF)%MkSlvSE2WHM z)bBJhxb-&P+u5zSfo6ZYpq@td=NpCE@=u7LN|+8A%eja07l!4xrB8NWV%zE&M7H{z z-b@nmPr*+oiB0*XW-d(K@~Mf=EmD6wO*kV}u&cQKv0f{;_^EI?Z|5yTI)g{N7mQQ# z9QWMF1lKjCyd|>f`XqUvx0T+QNWfib;2M)v+3}}y8gxB7*%R_Eekipte8)3aZ)gqy z=czZnL2+8K-Z*__z!N*%yeKcGN6-t*=5`sTDpcs*xb@lZV#={8SxcCa!VbkW`*=@A zItnM%{ZJEMlm~e72D@3)?Y>g?>L>zjE*nLuKbZ0^PRa69K*kWPG*p=2Ao*;az6QQz7cp0QPC^2 z7h9)dmSyB8#F*IZ1RksekwI8zh)(RIwV_iWReF`!uXPPb4TKsq8ieqxP z0o0aLD3X0Dm2%uP3rHvU98`T`o79S~Pd+4SgN0NLO1@$Cv|vxSLhFGsKm$%@EjKrM zc1we~G1PRK;(}JTTAsdSHgSDdG9m|@EZsXCcHVMT|M`#o%6ris8)@gwGX97>b)L09 zs(V~{^)=y(-j{(F_8!-M;}Q{;>Irwnh#B;iwd&{>;0^IibqfFGh0@G5y)so_938dX zEUfkM?2>!ylkK8pdw4*!7tRD%Q8}!`yiMMdS$>q=kXaYb!wFP%h4b9V$l+b6r(55+ zo1(xS()zFsq!*q}YFo7v=KCkfJmx~WzJW<9(@5*DA95ZgYv`5DuSA5oq2yz`!9ihz z{00i84-)HIr@s+jc-HBbG~QveTh-lQYCMZ9^86hR3BM4VBH7$f@|OP2Hu$;9IqZdx zi7wgmG)5P!kaKr_v)_>$9Kg zUwkE09*no0=5=#Bb&=SnA7Qs+E2(6YLS40=`rflnzC;XT+DVoxhc!%TKtIPyttE89 z9>nUetiH}zRfZ?tX>x?Lhj_r%GTtLcaIDgpSWAAS{aANNoA`8Z1wD$mM;})&5|6Cg_CVKk@B#Yj<;c=@9a(q3;)27|q>+)6NGsvs?3vRy;ZCTlvn+Y`llw43qJ zIW&dP*YF`B9>_md zhS>Sodr~UVlo=$iHsay3UY?f0p?J+a)uz&eo!j*WRGN^eT?Nl7p4(5pV4-A@Xm@Ug zG)Fr++LIO1&Cn|Xm0UH6Ag0E3%_-ynwVnC5QI_v2?o9{r`#MXHbMe^&1j>LOA0 zXk(9FpE#tJ4~;ZGf@wJniiD%kpeB(mnN@s2>NOk)I63kV4FLopth7>n}iQ$_$#=1PNWY-nXH{We%ryTd=3wX{(wemKJ zzZ9LzIacWD_Ih4p9y|{Ws)j5Wc~&#^gVxdJw7ZeqaxEn>yd>fb?D~-$4g_bUACclC zVsLcE7Jm~dHPS_X9zKE@^gz82kz%xwNuxcu;}w`j^kyywsafUu!|pfHZDLgKX85Ij zih3W{J9e9EvS3F$V@A0u38RJfd=5U9+d-e86kvnnh%;taPj~ z%pBk^l@-alCsqtzOy8FAYv@y?czAhmZ{U37hB8`-C}*^)a0y_cpPdHv=nbL2^Suz| z{wKC*^mR|6sQS?<(SzdN#k}&&gDb>>=qO)jN48_ABbLQe$EAW>KNDxd82C)hH9kW5 z`EGYu>GahQf^vh z*iMX_Y|o^+DtJU^F7KvTZ&Z`$OVPgA#Njg%I49SvntFx66dJH%^eSlP!f;;^U_e6J6%*)tC(M^3@Vn)U#<=7v0+tUACJ%?3 zWEEf?rL#|GWoCE!xhu1?x*(!|J@YAd_^eVdP%2ByJmBvb8L4zsw1}u~uqm*Sda>tu zr}HxqmW|PEaQV}&cJgZ&$ zkzjSWl++4e7hmW#z>3<33E6Y*f}_53oU2Pro>;?MJ31kDSnSlecJalc>-erCD`K9v zF>*etxHFNebI|b$3WNVq2EC8I2}XBNe}$yZ<)|jtBRw@*%^@dA+d}8En;`+~R_5`a zDS>%nC4bxRNm<20=R)y;tyyCB?%*)_l>AmqK=x%rk_4}44s*^i6U>Lkp6Hm)(Nnw| zqT=AtOvUzzdL0#v85H~2mxLs>zdZGka0d+(D={D8FItsZi&@}TYaLO<{tJ`afYw90 ztL!%>Dj&j=!bYfka1LA`e@*X~xiZpDBr*nNR7Dm_{lIKYZR_|~ie9BoxO2p(O|W)a zC%~wG#IE2A^T!>pqjp3$_O$f8kM0*A756>*O!UC$;nDe`e~pQbzTl~bw4ueWF31mQ z3-t-f&f(pR1jh7stC;m4bEbe+6G%*Rqn@%3GvYJi`^bUt(4gdBmN~*NhW+88nR|Xz z@iz)b0%NkLXDHgIQRZWFq>X+O?Vn!rhj^`xF6w3X&?I>X;=Gv z%c8zU_lU0Ko9Qm(F6f@%wxPlAz?I-0u}c^pUbe$Y8m#+G#1ZSH5e4)lC!G4uslyaQ z&aJeU3F%+a3MB;Rgf~Z?`oQ-q|_YBvG6cei`d9+WMeB?F;Ks)r3m`rCv^YVt<=Zy6Z_4ILma~F)- z8GScuyLYzdov&Cl@0;&h>Y3>2fvms(Qp!}WGA4Ni*r99;6HnH%KM^*jvBS+j^?~MW zpt$GMC(wY6R)FQmtb8XjDts$^C6X9k=bsoL#UtW4znEPsoCgT*(on6)ACUxUftn`I zRF>!$%p|1CPln&=WO^9C)UnaY0!7YrO73bgXQQsUXJSQHkDB9|;a%WKbBEmhUDF&R zg$zeIaJ4k%9Qz)QvtIDcJHs30v9%f**yGKdnEAg@uPIYhj}|A_6hDfpq=mCXIgx5` zA^6xoHM~#i6`Jp#6L>CFk$;IS4`qaDb&&oH$vl0v6ZUDiPJ^@0WV1OqKcDQFkCdHH zuJ-QVz2%~td3Cq!`73H-RB3M|cTV?8PYKU7M=^)(*eW!D%jH0>Hd~3_3zlpWxfniV z8=>4zwVql|BNMER#_$nosQe#$?;Rag5&sR}vb|?FJ%NPYtMsOTNEHxJdX=tpLAnAW zN>z%WGzFyz3J3@YNH5Y`AhZNXLJDcS*?!CWnZ52C@b^6Ld(Lyt^WW^8y?5^1Dc||b zcgp?F%y-`QRP_eDjCXWNcl3eklJ`r>xjN!pr8dw9t}C8cV0E|0Zn4X*-tMa2WPeKF zY@nk5g-{lIV7oBya6!#MEr%KVcBbCuqeh!4!d$_6&lYR_%G47&YZ&V(Q>t-`sj{&l z<}zFBztKvX5jY!rkgB;ln!hD~&-{oQ@&UBMJQAu2KSLkEjzA~u%9`#Q=^KsRjukv@ zF-sha+RwO#LPJihQ|%hyIa)H-S$BD9l)pF+C)D{pg+!a+aX!8CwmUlLRF02 zzKr%SeFo+%<{J;-T-2e~(bhckBhy*SW$RZs>F*Ql1l2+2-r10?)o2Q^tIVr@Pu+^U z$LwZX@YT4z&>8!oG+*wHomAfkz5XjWJN1nq>lb}RzW04Ye0#i`JwLe9$;_8~LP@l9 zxGT9N$vL372s`V#IXe{nSd!rz=KTfxRtJ0U1%HJ$lOqAYGzj_){^Dk-f5q(LJNm=M z#+JVXE7oSh<_Z|A-wN z>oN0}%$%22NPcKOT#NI`@`Mk?O4yNjAvg*1biW6W1&?80F$=vQze8h62b{FF%~Rjw za%x>}_x6&JMf071db+tS#p8+E1&*yr>_XfCT63&Q9fZoxK z{LiZ8s$QCo1{hr%<0wm!*#<5gfz$bJS?XhkWsz}_X{xcBej3iNi^OibrC2F3pO5BO zV7A}Mc3_7<)8sh08P*HO$^)d;(iz!?{URySZ_*X?Ooy-<`jR{y>o5C4)5Tj@2QUV$ z-)v!R@WWuxU(vtR*WWi3I}ERtOm*EZDN}L;tMlTEM>$(#Z25;1bI~{ju7!6T);;|0 z+ZLQ9-4JSt^>LD26n59qY$_oCn8P&YN$Wwqji%V{3w%a}>Ljv6c+AnyfZ` zj6Jl&RH@L7A!C)opOBsp#BRfb*gxG4cz1<1&TG&t_d7Xd4t;3^JBeuoO_JY2i{?@0 zHdY<{irpiv@K&H}P!(7YoygJNt?t&Ct9$57bmlsbxqd9_UYJ?b$7v|p z<*ZqFv8cek-*?eH!S$C1(n_&`FbC@$w{uy1E!9`*wmKbV%XjJ><`dSt<{n09ImUUp zIp$-=Td*%O^)v*uqjmY(KImi1tDM*|RT*o^uVN2lR4= z{SJ0ce~48K*STE2qpB)@g^$3BnL>UubPZ*w%3)v zw~#J(M>#y>zt!y2AHt0CQ^RvJW9bU5*z3$LOMjg9Gu$}LRATCCtfJ4-tVL9x`73rQ^MYy3E|ec&h4duoKpu!S(z1T<9`-4|BiOJU&;7 zv(WitNk!MbqUz4?Tn?-dIO0q#S?pfoD(5=uUgzfmHt$+*0?H;9F*HD(=mL#ab=87a zXSj;B3Ra`jG|}A3w9Tk9-!|Vctu)p~?QUvx>c(p4LT~+k(A5#E7s_Ej;&gD-ChSQY zi&JrTOCLg;@K@qFAyfJl8n32Ey|B)`8#ISRL!U+p#?)(|lfOvIgr$8$)N$a-3h;*^}hi?ygsIw4}YOvMbBE+WFjB z#km`2y!I)+;k3CHIr)-@o(9de_gZ-@)puMyd_mAnq?VHMEENFK7z>18W=l zLf3I!X`++~%{D8asi7(4$0nr4jLXq3}e&|I@q{RebvL~5sCl}KOhSxp(O zU0Xp@P3^;Jl&$zAZUDOnE8nn274vkngVh7Pf1CF**1JA&KP#z$+EvCCSK@Kqa87b@ zuCJXnF-!k}vx%#=vxV!3` z^f~k;-NR_)xoD7_(Cd6(yeKXey;8b-L24*RVU5~+Xh~_xCqa|fM4Up`0Vm&fg>J`y z_6?j$j71E(h1wLXFL?yrGh6rszAn}&1mwZeulRmH8hncL{Y?IfIBh5q`pcr+ok|Yj zyv{l$HC^>wqg|3S9UAa96)(X||9s~-^pa;>$4feU7W(Gk3x9>btjn+bMLURz)IsGtUAg!wlXa>%`r!qQw$vp zs}08t!}M=y=R*hh-|CL46lk;g0{vD7T0x_{8*|pv#pOaK^fwxC-f$rJ7f!=;h&Qp% z^iAjhuaCa%0`vfFkw>sb4e+LKy6&cOIw`z7|8M?lw5 zD^(xtUO0}GWaXL7QjyRK`kuOAmBekVHC==ir+MD0zVq%l?|jcdZ>IZ(X99Gxl(GCG*c3p?{Qq@#{ zsS7kWHIdLUwox}u0kDoo2$$l z`FO0@-3%RD{h?9)2JhpuFwRQBipJ;Mb*y1kaYwL@vj#MB)WB+}>ath*09sWyi8aL9 z_}VxG%{5DcA3%S4TVZ{0Nw6u_SS$+u6xFz5@u39Vc`A$jgCVs|C>?o5<^kgU)*?vsZ@3zz{b>`Jg6cpsOGRd#KlMXMOp z{lVXXjP?zj`+W&CKZ9Pd3H)H>pbaz<+MsvlQ)o`_!&SoivrX71Kac&Atp>e)b=VSU zemV+$M?Ye1;7qIy90Q$Wv$0xY46bva8R`r)jS~%I&ycPIdvJpII$IjjRyGd%T(3i? z)^YYMVkEI{0cC5UE2s%{c9cLjP%861w5RK#liP=0ud5sn&b}iZgTD2lSZ(V@IX{7% zeuelEzKCXt7)*jDD;!&L&C8YF=?T( zycKj%&Bc!SeOP^{#t5eeBxy^bamWKbO%~`I_=H^n+IEA=yBN=zp$#{dOT-ynwV}D^ zZD^Zs3DW>N=;Fz$JkAWAdj;%GwdNy5;giTAXC)#VEnvJsnC_ z2W?g|G&_+pBd62+4gVWZcJr8N7%{eETHy468d%E}8Om({HRd67+h^f=4W|}dhL-t< z_#Xoe4`J@0J-LT}hMb0V&}hxzA0ipFuRIfgmU#)>pvQ_=V$(7fGAjoCbRLu*u`}?) zz=db9Wd-3#2tDjRobtfH#SIq$`zU?5^YE<=h34@_4Z9XTSy0KMWb}xQxbPvH9(OHV z4524-wS|0>v~AEkNc@LmFn~IJh&}6xzN2v{JOwL3KtcFtBB8hxDfw&Fxhy@14rhD-sM9VPXdpT+{tOP5Ajb$iEAVy~?#v zf44!GTb^8A)mqJ>Oul4?@}0TaXsM^6hOS|z$RA<7^;Wc3(Xv(AC;x*sZaNbk`Zm2L zzmK+XC)UWHkOzS;H$yM`4e08sDj#JgLj!n6tX_|V2HW{K)9k8*d1ds6-Q_Cy@-By6 z_%7)bzQV8JoTGO!(jSlAlk>5sYddBd-O?%KY6LW;EW@gx9_T@vVxNcC4;M zT82qzn^B(4ZeeyH#zg4EyM`Q9M2oovBf(^p zVh3mtoDPkfG3W<3px$kGu{; z8PoyV+o3&j3uZrBfIbt(2@9Czpne>rp6SpTSRW|FVZ4-&UGB|M@@tqlxX;Aihb_qM zLDcXR^q1oxPiccun;){E0T{Jgp;xYov<8~2CX3af6JVy0FH9HvU@SWvQl9-nC+IEx z91@GRK^^wZUB@?Lv7g+<0<;Qc2xE0)PjGT93E_@@XxG(jD46>K>h4>I7u}9)d@jWpfE5}Dklc5vz zj#MBGM%^8T(Z&Mwn*DGh&94|WBymRW2khcV!M>5^$RX)hgDA~nv<}n3KPq{vbV2$V zS`I5fj}k@z%uM-9jAW8A?mUlgyRK|UNCYmTOafdBbb~#)(a_m5fv>>jVZUo4MlZ*} z@q^KBY?Ei;J7FqHa~aO^i4s%9n%E2R6;PRs(+8g5Yp$C(1RB7GVEpup6d_F))(cK? zFQimwgvvq}_*o$?5G=ysU`1#(z3!ijyPq@Do{Chu?)aeBpc=!_f!{noqWwoI1vH}GmIO@{uf@7Y~*?NBZ3L&`->w4fFxL0jBejEo0BujfN(#Q6Xk zq2@sr5zP$)H~zys#E3?N(;^OFOxljQ%obrMg9-g_KWK*8jWd$;XjRr@7PbM`6sLok z(T42-N6y7~>>uo~D-tx48MFEU=uaCbe1$KnFT}w(OXzRtsr&$Y{r6!`hw$wu%&*Y; zITdZjWqCSU;6jXX2Fg9L3wWZm3*QO%gsi{^!MCNI&=U1I_R;r|4q&W??I+Uv*fZ4w z*p%VlRu^J#oErP?uc|+R614l!5Z9i+%H74d`Y^szdgJ7PMbcbwX&S!VZw1vNBUhID zVK?nO?B)LuxV(>Eyc+t6e5MHbIghre613V@f(F667*8C=|2a;6*o8i8m$U+UwU0={ z<>$yv1Wt822C3{0`IzJs?@L5~rwZ-qaO`3o0E!l~)lo&Yzr*|Xn&TX?L7t= z@)_t2O%An?7o@Y&3AESA*xhs(eTP=w3%(JcjUWT^>9ofj64t%f%Bjz^vl0Z*^6Rr-T zXX^|-Q%2CpqhC1>Y2jLVCnVq>LEc*heG}1wRT1CCKE#lQ)4INfBzrBiptX_ii`kH6 zpFoN}c|5+6`lFn_!hA;q>`)aUd0hwIm<&Yv;3TJx$Z1t@%{q1#WSqTG7BQ$%r_h^i z!H(3UsEs$!0>ptI0`h1n9a=3cz;GXWuW#`Mm4b4r3mv=r(5rUG=<*c48OEYb><{_F zV2l(hL3`~MaCHy3T|kep8(&BX*o~10Im=4sA!3d~JuX2m7J{Er(dV6GmZ9gu_bSS@ z0`kxf{I~`s`5as@AHC-)jHD((`{r~QauU~kaKOjlJqaA`#7S#EqNR=j{~g6X(>(S* z`v^H|jjy{&;MO(hy^leEoF3X}mmucROdXuQ@*C=U97eTM(O0EIpXVlU$}?sFJ{UszY!y#i1pjCT@(c&u9b_Oj_Jmjfe zG4jbpdGEpKI3FB5Q0@d-bSnJaf##E*pmZR7GH) zl#6|udQepa|J~6N&V~$NCG7u%b}U*^e~+LKoQ;tZbmgHxJc@C-N1lfszAMVajroxk zkQFS($yyJf?M?!AgQcI)HsDJZngQdH``*xxybv?lb)ZA@ERgHNIuZXql+rkid&_{g zD+7fyh?R_}uVGx*S>7nkLY+y2=Bha0I00?!H)t0$kY{wo32Es-K#%j=Y{<{M;HkGz z`d!iPPCOTelv+NgZ^%!N#70h7Ue1_||U z)UNhexjGhj_QqG*F^s+fIN@O+T7b_Gt|P{3-=n2@4F2^)+SCs52$0#FmGb5C=+8G} z^fZQDi+*Q1v}{{2-;xFyt66%96G#4n7Ot)+?{nzSm!owUgxsveX!8J2nT&r4Mh_Jc z;(dG<^hI8WBTby!1(~uNJ-8XAJQ#h|NPJNU_)=X2-)*pkvb{M1VmG;3rm+T1nJtP`>jOrGlrC*5= zc3sGKas-%S~W&7+(n;gq-{sTsJZ6?*lWXf@iSCfcC+T7!MehtWc`MjJ=I zR1X2!!I03@MDBONB?f)IgdX^B)Rs!f&qMSiV?b49;QArtKNX=HXC=PFvcN&}P?w&7 zo>Yt$l9B#el=6}g=7&)dpCc!P&gpV9^h_(khYRr?cMDvSixQjy$zc)QPhqA+LwCB`3}(j2S&G*;d?G>+E~2J z0_SPbrVyE2E%0M^^g~2r{6-)!0d#LcTih8ndl}F>fqLBtRJKRFK75fj0pC7EkJk$A z={D4)*=UWHA%`i5D++R-GvLUn=ox-SZi&ny68^GK(x=eB%>mZaKxI1o^g({ike#&! z;%?MRGkEAA(5s9dI0m)<7~1Zg$XOTU(}2;!=jcyIgSI@Rjd3{ApNDqt3|i`VwjU&) z=R#c405q_8*B>}MMy(}sE*AXM7T7kaa z5M1H|YNWO$p>%hFsuqZ~8uF5Xnz<6aQyx-Z3|tyuj1q}F< z$H0dY=F116+@B%8_=W}&yTDg@sOJd?nF78m1JvI{`_T${vVf9eaPl}%xC%L6iXLGp zxUCVwp8HQ<{Jnj*nzHBjGrBc>W~FN1#*;5r@s(gL(mBT=qh@ox*t z6schXYC{z2elxh(!8Ht0vcQGN8lytB>LL0zB0F<~USf#+NkX|)1^2ba{|%%Og|w?8 zhFS;I>_Bf_-lq3%Oif%zmirD=zNAh>C+6Lrw(#D4E1jU{zUdpCyTnC6(2I2&7 zB0G;kJVe^8$S(~cid-nKG^EAgZPk$fIK)b1!g|D$3=@SIiF}jDi%oE=jGPjgsv>zU z1F9N{Fh?@(h7=Fn3j>xvvFhovEq^}C5Bs`&k50z|82osVDNr%Xhi4>iLAX0o5 zDH43 zh%A{%h_7Iraw+sD?a5_4lc&pQofUH-bU%Z3>=fFklOdm%{`Geb;fbBZa|8biA#(|F zkol(4tI}4H*e}EX4ZI;?$b03DVka>WE+R7Sd$^N%8uCn}*hG#^{E|B{DIs@-j<8gl z^lD^Yhe)=SIUW*^(%TVL*xssP!Aj6UN6-0mWMm>srXhnk<6hG?j{_p{0XlSJ`T^A zgsCK)uNcDD%A5$vCHa%Q5-Iok*nfPVj#ZAkAx$k zl~|PdC}jplnZHuf3)@3MLjH-nlBVLOP^{3W*hyMSc_@ArSH(BEhT{y!626z_?o}!Y zeWkQ0v4le?Pf9o?RwX2PuY?Ly8xB!Qtr9kTEu}GxX*fLbp@b%W!oN~m*d^>Md?!8> zYD#_l_nEwTHCClW6xVP(O6ah!Fm=k?(sF$@9+GmHKIN|DPH|UUNR0{q6c>fIFomzW zy&5+OqtK%GP@a{ThDmeL+hDeSNGjgq#KMyW4?M%Yg{ z-BK!*SjcnOEld}A^Qyb@{#6$xZ1@@uuROo{L}Dxr9k!R=UrpzK`g_%-H12STgi|lQ zmtsaj{;xtSZly6O&tbpen8G2$cq;D1HSGUYTj^WkTIwSl+w1qz_ocRQxK~5JYA=l; z>>6$vlsLjX7WPM6!#;_d@;;ni=^J8uJ*IFB;qc)wiuYa<0R=R@e!BTX?oR9sr|qH{%2?sOPI#eaEePfh0;)AAB2uD zE#W?iynEGWX{@Dw!{J}Ozn<2sZ(fftOpW3rocinjOGCU$$$$O)H_xT?g+ql)<-Z}7 z_`|-Gw+dbVJ9-p9uZJj&_4PcJy1!~u!j{HRY9qe>cTc5xdNr=sxrT&&_5a`4UJv&Qm*7J98>8N@f&viU&a4A&80p{(<*g)-4+h{ zYCglEl)FMjnA-5W@H2_+)v)Bb)Q@6+_4&W=$@71I|MmZRjIa8B-RJ91uloBxw<~d$ z#$0Ms;tb;#etJEnuy3U$dHo%Iq6km~C;}7#iU37`B0v$K2v7tl0u%v?07ZZzKoOt_ zPy{Ff6ak6=MSvne5ugZA1SkR&0g3=cfFeKReTOX*iACsZg@ zDQ}fH6xzbEhu;v&6~AHH!zn71k!zT?Fsw;MckD*N&Ka0 zD(>V;C{V&HG=jCA|b*)NbbV1gxyKbm6!-d*&%d@Ee9wP^c3Vu zF;!g28-hJ~$`1J>xaa*_4x}tdX%vRaiP)SWI|)(v@4qDE>I_}o|CVS8%9a>X_N1JZ zk|v?apM+2hDSr|ZiHn$$SM21S!VL-)gerylFntQu`JvE+HpPd+8N{!`9RwGWGlG-C zA7OU|ErnMU?9$+i;7@rG84XVgh)U zLT-NC!|kt9uSrWN$)x@(Eh%Y-NSmOvGNjcAw=blPA?+AxPslag4u#tuBV+#8U3p9V zD*lxgh=d~T6Y)>Nk~S^e>Pa$r3b%ZurBT`=awmq+M%pU!3%?~cB{vG?;e3(&3;*U^ z2@!73l+ffF_Cb&d`z3t@iJOEWSP~S8A->6##6tWs3~AR0-sFk+BcYTuNLZ2+@*~J9 zxl-~)FeP~*xmH?Sf&#(D2t%+XMnTEOSQ#6PIdr!&kucGWJ#>#`;+RC3viK+eYs%q? zJSX6q0JpOElY1qm3jURt3bG-J_!x2Kw%@M z0q%qr;+xQ;ylI6u^_V&cQ5CT^z!SMv!4tu#EE9vb1c5p*twPs2z|oAi1nUI6uZ0vV zz!n+8kKkJqPto9vs_tbJ!VX6k;z69@;LS2VM8js#Ha8NJ)0X-z`@`9{!LUfom^= zx&SDukCa!-Z-v4JPe@beEdg5kNAd?bH??aGdpXUGZc0d^`oiCHIilFu=F zlo!Z)c_MqC9m_?rSDDJPk(ATk6dA zmcJ2uF?Z!&@+G`w*h-RCuFibGY+&ZfC#8QFlKUa@6O`63QWR2(LX2*pF;z+zPe@jF zwscN1%AZPyr1|19=1pmoxKW(n;kT`au{H9zR+;Rv#m|s37uaLS* z_vAumyWkPKf*)$jZ%Ixeh*-9=e@c^u1c_m%@>Lm!G+L;_Oy>PemgpAhvN`eucE7Yi zT+WP^IX0c~h(F4U<&VK9!^G9Hj{8`yDfN)E#a!ki_FeXj*j%bD?_xJF?@0rtTi}p$ z%yZEsWpi7ZD||KCEXHsHxryu=t_e~sWIkp(uwTn_K+za*bCkSCoCt0{2wvVJoskZ) zy_h>(iCkYSkUDTbaTnwP63aA@drMW=L^(tHmWz@H$!2bul*&wDg3Fu=s(X)3C8j9+yKGCHD+UlEnJ5D4BoM>el5=8aO6Z+R0_|KLvY%|8j%B zGQY_EnQhVvq*hz@1K}oYbE&O-k?G9(nS;SOOdP+LImuj=Yol&_FSJqD)hrEk4Yp@K zRP7h*3ERNWgQc?KR;G`*gDGOCGdB4XVU~QGZ>P=lH3-J3pL18`PlZu(AN~i?$3}6V z3I)P@%xV4rP^d4iQ@z3dD9s2QmW}Gh@<&oiupM)a-7c?XT8LGGBKreZiJ8jo_t$59 z>b9!g0XEQBbB<3C8j0OmuY8kjA}wQYvTH;QvtF9Q&p_!nWZMVZ2gh^s_%Ynlz*3Z-#bE@ZUFzIFjDF$Xw*%ZjlxhqLdxToFym2gXMz{%Xqz*O z{ez{4{5o}O(aNMU6S$F51Ez#4kXB2-G6r@gQ%P#e)MclmR#s!L@MW1s@E6Os73WGh zs*mN}(l92Ob+Vb#V8+E<;+`|xn75?5Yy(v)vqW;sr?_0HIrAPcRHp(L{A;CZnyTvazK8yf>N?^a z@ds`&YUC}hFWS?cTrKoc{W%+I!WYb4_8qobV57W1(_Q*goGNXWo2hn4qvfB(+Wa=I zbs&vh&eai*i61a7_6oZ~7T6f^sl1y#&&*KuL`yu0S;qHb_H*9}mAR|j$Kncho-|qB zuP)D@^YoTisP4&4gpKSlrlK&6n4kW~umttW{ML%J{d)JJnP8Puy<`@9B8y&)_BbJ?Up|Fgqu> z2Z&T>mNC!R_RN0iTX7z{Oxh&$(enH;Uxx5h^%Xl;9x7GkvY7$iv0R>}C+m|Ug!d&2 zx4^%d`ObJ*e(W40E-QC5ifgNllf1VM9pTuLT-_;*8=Y+SUVd_gNmg_AK3?@ryd_&DimJMtPj8Q*j>u|Hx zm83?&%cySwz9Rc5_?`HNkj{+7SmL>-saQ!IuJdRY6)!Gnt??K}vuS~#@4EH^{|_@m zs3{Co_tnnWaM| zeYm=oR>4Nab-8B73EVhul((`f-*81(cyFv<6fS)<)2iJp~0HPsYutQ_#>b91!T;3D4<)Lg$jjK9kLAwKZMFvZ$GRI37< z{}1`5A%@!fco3rJsZQ*tYt^7&9`= zb}n1J1&G~6JAIBFu8C4f%scFd8cF+tf0MVXTWGuMD&uoDMfZy;UfWi0({ASS)v@{| z{5@_g`z3!&-B;CLe8}YR0=T_}sFybhC;jEUR{yHN2f+v4b)Mf#E_hS@qXVlv`(2lu z6TFFms-BvjcU|jTn>>ZZJB#%0$0Z--Pkh;=IIGC)dR`>tb3N{LIz$MQVDmaDXsh+~)t))5+IZ)6uw4zg$1b)Xdxm_3QZ zu&2C7jO2?n&{b_6M5y`%Tr!vo2sdLKg~!z z>)T+zZ1`H-!!NVtMO=?CMox+?pExw>x1_kZArbeZn2JAE4miGHPH30M4vd?mT@_q} z9*gDvWPX(0f%kmd#g=N3*D>=|H_dsbWtwP1x41pY-^VmI4z_(%u1eMR2}{g>nQO;J zB~^|6!~DWWWbqTm#cev*s{E))O}By$!_yn-JkqM&3uz!8*h({ zxn1T?^6tdSu~nkpk54Rrw8D_sm%1`K-rhAP)_6>u7aYW@(00WHz1{|*$VRbA@&-On zzgyQ=Gg((7VpPKVL~YbG%Z;c_<=(B>G4TugZ2Ql#8HoikXUu=;ADJ54cyl>bFY#)y zmAr%bF%ae2R(QT(XVH`5ck+j1wM=i4-uQW&bXDrD8xwC|y;pdn%;{Pe-Pw2DpWaXZ z(|ma<)5_s0{QRFCDaTdYsjwoGv3*c-gK)H#g3WgG)e8P#T+r`7;W+?_luxS-BjCEw_LwXyI*};f7fv<_OGazh)WR@W8O{h z#XpEGiaHTFD0*mYM9gOqM$03M%9Ns>!0zOxsv*1Z+$yM0{JuA>q;by3Cm%d+`t;?K zo+*27M&0Um|t>-;;fg-+A$&@n|6Tc8lw~N_s}tHNN9+?(c3cu(vgh ziZCSF6K*@2*iEt3%4o{oh=1VNYCdZU*m^j2nBQV91vs8JpVIdb2K&28V>P>sf1u`l zt;*8hw0DS}8d<|JGqP=LPW;=6wd22yy%fDL>QdCjnEBDmEv>ZE)XNOnh8euacdg`( zC(@tk8Rv}2d;aX#XYNd0X6=XNZ>HbKNja0^xfDD*{ISiq=0)IyaH693Xl!5Kw2K$7 z9pv^Te`EM3w_(v3YrBN$ma+Qj5q+cIiztcsIqq8g?8q1PIgxMKzciFJ%(o7))>qvN z+++G$zA;Y`7kewHf3y8y=+3rLU$P8|2wD>@TkTC^j>m3@j*I5vFUJi6r3EpcN6&FI zvTw2;G&ImJF)h$dmcHBXv?e%Z?(pSvTbRCsjiUgwJ=PBp%II6JRk z_?^ICBd<3We$%*K48Ah>@t2xO(M_cMoEd@UF}mpAv{Q}c;&#UVjdVwsSyt8`8Dn{9 zYZhg;_ck{-{Gz$U?dLu>O|Vp9-u4_~n>sQh4y#(SoeT}_jU5NAEA6J3?_y6!Er`mA zHi_nA9zDqBS%6jrit~znF~?e@3`;joD-Boe^5cP}MpA4EDBddsGwaR?Q{VQ{&&U zYsxN(UT@rCd@pWQ`3;GKZKEu+qISpDi)>?CY;hRNsTXoz@#9n+>-AnN2;|)>$|~ye zqS1rlcOp_6J-l&m`la4ymfgRWuTGCW{nfeNxz*&o1rc}bcQ?CQSq`cuy-X}f)VGPU znI34TSxmM`mVJ)ui3iIFQL@Eh{>x!NFEYY3S@XMkzV^DQuI-Bc3?v!PP1Wr2CcpZ9 z-K2>4s81aYA{IFw#|(?!>G&dgRzkh_M-jCx-#X4jueQHqo`zA!S*{;{j&CjWEcCx{ z=V!Sl<$d$ugiQ1W~=h ze#_BO=Vj;V4m%#jsT|SfFYU&-)aVWNhOws;nW)B=mbOumk@hiWlX0!4h_%S;C6n~r z?Z}&zbuBkD_w18`J3a3Gn9ASSe9>~({30%`-T9-ZDyLNn&Mr3HD!!8Za-iW4)t~}Z z{$X)2`X-;TpW#j8BkPr@nsK&-6|qes>)B^VI%8Afo<=>kRMzg-PI5eo{mEJd^2X0h z^&JoGtxdma-qGK3{GPZcW}Kyx^|$B_@#!%O9lzTbI2PMan8zF9tc$EO`Of|`;$l^S zaO&m77n5Ed$&br?ka{F#>-{-*4L7UaJ(1>r+~!Kw<=>xlFFc<$;_hcDopSmG*9*0a z67pUCGunPekM_Q9wb5wWVmlExC9z6Oh9%WHAvQBnifvt$;b`$@abJVP@`HJBeDtNmt= z-94o+IWsA3Px^xA`_o=rpK!5S%E|2HXSxdy&V2Pa-s#ENc1PK1}1nDw6W_QI~_w~O!0l=b6z(Uxy%Z(6Ei?Th`Q?C8X0ar@&=#7&K_oKQWkVbmJ?WCtI)!_L?wOBHJqTWyeK)$7$cgm(SjPa_;8M zD{bzNPyhIq?OdfR+>^ympWilKeUXy#;!;7U7pBa2^P>U>}_K|NSN!0(Bx~sw|9swus1dA)X%pvQL5-V5s?w~ z9U~kL{Ch=aNA9yfHlH?5F+a1uV|>9L^iA-M!Ki+BUZeEY>AGhv9}l`!{z{z()@L`b z&;R?QOS3X;8A;c(FZQ~dnscdu&8qUOZov)VsW8uP=N?&3nf7Q;S+6G+CQnO#F|$-uG?>znsP%Wv*Zw*?j8C4a>r)eO?&=md`j=Dsdu+NTk&w;$=b(D z?mv7n>9On5r{{i5jdn_|GiehZ%qzLU%@Do~bXO%AB||ma)|ek-_Qdr~?v)%Jzc)HM zDYNp<?aZB|4Rd!@)ekxw1w zh;MC&9Vxba!)jf5v&L{p(tE%3E)VuD*_xf6{{53rpGQBrccbAy+4mb}b-U|3?mPEw zTBDrw2Yb((uT04(<3F4K(PMj-Ca^@RB0OPwsmp1rn=VDi#C{UdA$nQ4?~4h_eD__d_9r8D3O?Xs0b^rY_w|m{yrJa9J`)bUkv3D*$ z>y|e3-n-YidwXB3$bavp;l<^gx1E0p9`w~FsiG#++As1z%%P~2F}D)BC!J255PvDT zOS#22$4*>1XN*lVg4QN~`? zGF(@pZD9G`wn87lu3|c?T5zWWTRo-{UGbNN@8%46dhJP{^lOjiUEh1V%`@B68n<#T zj!9{l?s--@t>w)Fcig#2?nBwj(i4gkf*#*&Ndx)VI=$L%wb!-8M_(*^B(5-SMxryJ zWm0}}gM`bGw;ZjL-YK)se%n^j_E|(NN0j54ZKJ8g`n#=@q0n^9I>=^@Xl;C?8g3jL zebXMJp3UTHM_4bLPH;zq)uh^dV|KB(igTq?U(`P@GDpY`Jbm)G%Y&+SlkQwg*JX}; z*z-!W>tCeh=H_K-@72AVkv`vZ%TxWu*6byo8gdGo26^@aP5+2~k(VNt*ek{-lxv;* zTlCfF&ytN59+x{Ab1LG~m{SS5gr$~q=H$pjHo^G0G0s|G?x*{}FkjzGf6hGI+(SQ6 zGeO_ZwA$jc-qVhdKi99d&(%MY?_f@>GS^v^BTOy1;BMvl*5xgDKR4^?gQugjIzO-f zVA`!IcXJ=#&l>uq>aA+mw?FClVrlu@oTqC(d!idP$(R^}^q#qMA94+m6u~(yyMm)5Zi)d*ZVp)wZ(_GU=(^+G6 zT@TIA`bU;=<{{cW{6D&Z)-R3ERXm@jsjuIu?Ikx5&vSR#O@bq^Nth8ZXVu9^8IJKZ>@jG=U#Ij&ARe<`19Y3fATzW{pl)n zH3+)p`Kk@veg2NAx2-`$gfZXRE_Q!H!`OqC0>`@8GVz|6JV$eT1xrtRT}OPx49jlg zXNFUjskZTkZwz|VBlQc-0R2VtR`WnzvU-Xx+T7CI&G57Ok;FDiZ+k#2wC$Q+zqEuYW*^7;GO**QX1 zgW|pgwet2BSX{osy3VU^b4djk8ypu1_}`RTVV<#yI$6zW&Kp)3ZyNd=7+YOiiq&tK zh92sD`zC;csLN|K z`hL2ayrjBf>+BD5wXrndY``hs$IMW#;M<;q%-5ci(i@)ICCk+BIKL?TBw%vu-Lss# zotXtYoGlB-`cey<<;^Yp;^oBRMaA6;R=(I-RJov;d#P)0&Nl^9z2}%Z9(~d6l2}R2 zekNX%_WC-BYRz4ZO6y~eFl!<#7N_YZ`>k<-E!lcS=Q4b09&Qi~*NnZapUbCh?X?%J zpBe|SQ?xZ~om9OHi!}+FLB?LHnwrlwQO2|EHPuD!H`-rRk5%4tY7`I1EQ!%VVHJ1B@U#ZUKTB?&YTh(^`Oy(&U(48?f)GcD#Xio6= z&D&M&wQY2j)bD5vhAe$Eet_zhs-1R*riV7#TgmQGNBBEwpK4cm`WSlhM}#lc5zcm| z7`Y4go%d7WFjq6Uh7bA=v0q3(`qMP0WwYPyzoe-owh8tUrNDDjx1tZZxvCwW75sjm zM?Fc^IB$lgJ~P?Bn2j&`(~#lqFW&U@)(`MTvt`98(o@YX*IDj_P9+}jc4KT@U$&(n z$RDc8`Cdws%jgc9EU`FzIK&sy!lJ-8g-)Y0JF&Qk+Y$y2V24o z5;{un=xe)ATQ>Roiv{}C&ZFA;>KthXzoW!vvdP2Qm_T#wQRa;ACVN@kTv{ub_(*aC6aaF2si2ATtJ-AWZS-LJYW23x-EOmYB#9OKz%sufwW_{`k^VBMFrmmOI zBwytBNdfs2O@x1eN{cz7iW0Bw&*=Oq_Rje~>27M;iXWJ6IQQvdxcz>&FKU;QoDRhq)x7E-w_M$m94CYN`0J zWr8>@*q39~R-srttcnpw^LK)83ftkkmwHfu)pv7C+)PNn<5lw{z3A1b{eMZbxHQ(x zJQTj*2XR;Yw|Tv4xG8u^ClrZJ}i2EK#*z4WWN*_7nG$i39DzH!1GezCk+o+&og_@sE>0e&Kr z!nWsY3*)dpB2laH{A?)pk5GT^>w+4#(wnXSh)LvbdaLmxv~QLiG&N%UfxFyF{(Juc zzNxCFcNgDY_tf`C;Iir;UB1U`81MPW-^+BRt^Qju0zrKMCyT+ptRok;xK9sV1oYcJFSd)f%pONrB48UkZHB-j?Ip-R@56S=zIKjlwDJ zL%BRVo6QY2VSmQ*F0<4d>q-s?)ijT^zjy=e_xweFL*4ekA%38ElU)>CugdV2W5(&3 zU_iCbEn+s|xL>PT%55u|&P>+y2_Dh>=&hq$q)zs*>P?baEaCca)5Q-0)!1%abkM49 zE9<~zN4ehKZ`9YM?cynI{Xi;nQ|clGdAo1A?2^v#eO2qF+ks5&)4+D&7n`G`n$%eC zCrwcK1MT@A*~}m(H_{GrrD&#WdwB*MH~KESD)GZ~v+&i~2J=Bj#bjZEAuny1Ke&cDB2QbBMBZ(9?k*zbNG^VE!H8IWA`CM)D8KzY-JqCazdv z4U9ft`5(RkCxAb#kgjU?X(z$L;&5TDG*SCZ+-wh5Iw*?xQaf4LCZ~G*4HJcSo@#t& zevD?PlptMqE^t59KNZ$F-#9%Uv$Tnqg=LQ4+`s6emD#qQ{2k9WVVzi4ykpCCUDK5C zVd7if!3pJlYW$+YL{uI@W*w}+|%9h^4}87lX40-NgV8{;W#CCmk;xy+-0eQ z)Zfw5`L^pc{NDWBk?-hc+f?4!-p@JHHqJRhI%scW``f#sx$ops~20h;rMoyL0F1pRfx0@FfWm|=@?meFqh+4S7_r{SczjmhF= zF@6Mh1nrE&4X@$ZbG&YiAzM37_qHZZOxG{c%bF9ghGeVeu8=2m6>ba9IYInZDR3|J zG;`l@pL1Pse(qfF`P06lyq&X`{b+e`*_5(?@}e?bdDGG^#mh>2l!ld#DE2Dck<}%) zG+UQ3sbI5ZjpuDgsDyT88#ZXn~oX%4PLrcnq^Wi&j_)pAzu3jR}B9POn29H^^!VCf4O&h-g7l}EGR!{ zf8?>aVy%-b6J58Q)ysz$#+U9d9aq@BK+O9v%PTc9^=8uYmqp2Y%SQ9OqIZez7MeqP zolnysKB8mTCjoK3NA)v>45Q%PTwLiqVn5|9wH&e-J)L1y(RFSJw^??{UubIkcMJH@ zJJ{UcC)j*h(@ry1dqV%MQK!8l&*L9!d~`qR{uKA>s(ZiXw+`;)H|e&Rl6^+|gqU}k zmU~|{8T6lOx9g_qXXp-zQ)P?mo^!QJ>zVJK>1gQ*mTI^y8C?w7`wH7a?X|5L%1{M^jnGk29e-MsiR?g1DQ|Zpm6^=MqIT3Db8MrQFps(5Z*ktpnYuF2Y z)LzKuhsf{SZ&^OINzU%_MD0cMAb1Y4J*bxNLvMfY6t85T?dHFAT{Yvi7R?jBfN!E{ zW_)hirM;~o{he7J$LN3d+ZC`yX795)y7x4hS~;NS=hzH$ak)^S9ktq zYhHSwbdqIw+4GY1L!5DasDTm!By_z<((h9J4(A`5et~ z;~uZtKF7TuneF;=aRIklQD8pm!?pAn+#kEcU8n6SjtSB%;b-kiolSg4?&(=24TV)a zEnzj9)_uV7*zuNgq;0cpvbz_*S@GGl zD&|JI8J-)C8;%0|_s;R~D<#0|1K(LbtBfy&dD0YESJRB^rS#%^ic5u7at(K1d6UN9 zaM3u}{FhfNLmMvDwbYs6%62Eqv-u}{mM|GS;1ZYWDR8fbwN42n>wu)uLGU{*K&P z)Wgbxql%l`Cl+h>8QK|g#h;aU=P664k}t{{IJ*d)yj}$i3t1jG$JCIUYWusqkL{kl zmiuSr9^XgXMpskRabvmKx~bmp`%m|tF1A%hdSqMc@)o6s9B)Le`Ks@H-?PTgg~{&j z_V*kGZg2S`ev2kqoWP&sPKjeQ_2l<$i>~UH@E)}dcw_Vt$U1u2QJ}8&FA7m{xA2tW#@}yN}t)R@+s|MpV+_+K6|tir4qZ{deVB$8gIWQRTr=D z8{F&dyFCN6cTB$K4-7vV<{Jg^o-?Ri3(p^ZbC-FdJ%#Q7E=qU97zrM$@!4a5b>+_6 z)@bWx+j9Fs=Xg0;qc#8Rm1x|hy{_G*8!sGiZ?&h{uQ-F;Z9HX)Lwne0HjXyj6n~Pt z$=Tq4kEL#MM{bs~UK%5Hk#>3>yL-#&nh1CzWjFt#b3^OHN(+kXT4dK|?rU8WW3It1 zeh>FQwWSn!0hh&35E7L;?j{bCZNGJ>vkq6?u)r(Md$KW3{Kxaml3iM6?dN|P8KFTyPKt@29eXKb(iLsLy0A-%TtDc^42q_o!D z);H9L=q?IPqy*Ol*E4sdvQTqc*GAiqi5OzPGJ&59e140^D)w1dr~SshKIl5T|P(&i3>xZC?0A$zuB* z{%hT8a|gdgKJ$%f+${UI*PVGjqC0|piBk9~vo&BY&+?TK}b~xNQ-Qsr1?WH_v zEUZ#16r#kHVi%zvcLmljFZ5{LcU_yLpSXDVH{h!15^sqYgwKVg+%oC4OXqs%d>?*W z*5}_9&%?8W`TF-Xhn4!CNcRQjZTlWaiQG`T)G*pO+W3KCi}0RQ=BR7$>l!GTxmY1n z*aj=;+Q2j6yKV#5UvorvSogbV;?8*VE-$CsIZ|FCqzg~P3*vggn|oh!yKljY#~>j? z>?6$K$~~zLv%_J(?&-}3!?Tx0hH#@sKb_z2KI@q4Y~vo_Y~g6`@#3#)U+BCvy~S`% ztfnd4*Vb`A^W2k7@*__}S10!;iodu>^OhK`iPmq_>0v$W#qu26uda4-G`~!H+DSQVChqL7k6#XZY5dVt?R41 ztm!HKru^aF;q-ET>sklbD+$&xjfB;cgN1>@M|>}S7pH-1!aK0y`mr(`W`1vYT31(W z!@VtymuA7r#~8UD*GWj@55O*mXt*{E=YQb`^Xp&*`~!K6G+463jJS=HR})s7m%5vH=1Bv&`tVD|4y*E#`N2vz&l>j>c&jOX&6-={ zP=37B#qIB@E}7&ac?h4Y8K(6T-zbIBOjsd)US1FTX9mObwzhUxBsq zVCA2I+bB1cCvzuxg>msk2Bz_>a5!(t4;Cae(mkdwmK6mFR@I+M?Y=mFz+8ulcDbjt! zxy~8jN^-AK+(JEVidNP%(*$Tn3D@Nx-5XrKo^)xO%z>@1i(iRPVfF4ZsfpAKo5#!QQ|gnqM(B{w7ZosSkGyI)xlR`ZSGtyKxqa$LTub%p^4Cu z|A^npujSUrzH+(zp^_~>kbjcf!1~H$SQmdtoT4ceXNdc_ol;x3!QIVsOYX{lFE-P> z5qAm?VW)$S)KHo#1<4=DA1is>+d>=hCvmKJk?)~=DE;7RFU864DPQti#jRq0zFd9@ zzvbq^@5(T)m*9rJi{pmM=~8z&6!sBhh+k`liqE)ia-Ms+%j&uzz2-87d*UJSLtz!S zRGug;gB7n0UH+mR9uF!K3e_? zthf=rk=z(w;!9w}@8jLDLuk2jK=FqCF*)#*A_?}=+)&!W-jt27D`XAqh>3(9LtWro z2zxS)az_3V>?7KsOoV&5XypfGD$LXpaO%w0!yShe*L#E(u%7Y{@TI%b z9%+a?3ib%Oxt;uM{yBG8iB-atcCcS)E|@y039aB7aP+0(0oXyn)ytc5v zCMn$C{L6i*d%?l@vKw?hN*5 z2tJ$wUR4UUL!eLZD&K;|76HHZ&`;yJC9u0?8{FyEg8fFjzz)N}mzIKz7{I&_d#{FY zAAluRKWDutFESPj12N7)G&D}nPK(0B;! zaNDmeQ`W*>Gbij!TMORd&qwn)+!mPKgP;cwLT}6gnldE@+V~@P2;?<|{Z%Dk?+DOs zAB?%jfZYeiPzm(K19-Z*zCw#yu;cC>&~GB_wYv+k7Py z_7RN%YiYqtHh{lP1@2GaI03!733kv00Ov5^G!}LdhE(jm^eB(PM|Z=nj2rL|*B_wW z2{@mFU#4NOLun1PZUMB60}uO9=?g6$2z$A%!2djxLHC(Z(-3TN67*RG8l4Bqqsled zGv@)jzX0D10evmtt4Sc`Jj@O!VDE1ouwPT?^}Dc_>O*BP*yIiP!$h#xGVp%%hFJjZ z0Uo^-cKA8L&uRCqA9$n*G${n=G3cGOkiGy(4M2_uLR`8AXz<)gG zhPwry0QEu8>@egE0O?qOK&JO084cOzw-tgJ^+cwz>WbR5q3v#AAy(D1IrYGu34}< zF%m}0cy2m)aVPLsJK&#%osHMP4gqjYc?WhwjsyzY3yR)c2WE{<(6VNbuMRrk-rEqc zR4u@61#-K!7An#lF?HfE`oO4&1+)3pOl- zot$Q%Zwxf`z!ENi;EwWKuz;z8C+=d!YeFPodBZs4q4#jlwhlDG{gk-d*AHN2(1sH5 z1KeFK0!9h6I2T&11B^OgqgsG1av9Kvk3p+^;NSzUSt@9-ub%_PS@5PCAU_9aa1V7c!0*CnIS(`FO~A+i{@6m~iTmS|faW65 zTmh`a3f*uYEzZYx0d^6jJ*u!J?v5`8>O_!@KAZ^9bl^ad6>u+qQUwfSE1+f=PlDnG zWIzq>vwQ$8K#$J>yW+n4ETBmQ*c0HETET~6E#Sz&ndAlZVtNG^)DsVk1c31YkUz#J zz}N%WYQ#@~G_j%%rDC)MjGuwK_bIjo%B7Kjb@VN)&=PBihT?0mSQ;1u0Y?r-bwKIl z8?@UWXVP5I?*&MG0dmn#h&x7LK)$&Z6gcK7E(b}(86X4bkY6s;vA7Nt(*Y?_6O@Ru z(T3P+ij9G`Lkt$@0VQDF3$QQNQ9F<(wW2SupU{d)AP?J)_!v{;1;FpZaUXi*3Fw|w z!Q(l2?OpJ%8_?@lpx19g9x)K_K0vR54_yQ=LjO7s-?J4SgvBRxBoq;ukV6? z(1G*VWAKqD|9Uz45{_6*(QnZ2SWFg-zL5j|h+_#o8$B3PH8u$w2e|8=;@D7}2TCc{ z4Mw}c5r&a8FvUm`7$1T~qQLkOID2Ef3X1bU(S0xi4@Q8%gCap-R0yQv0TKd70LjQfDU9x$2@rWhLtF)@AwMVi3*(Ni%eV-8`MVq^-6ZNcJKu-H7LBh3wnNs%l_ zYoaF(ERF?A!**aR@Iz(HV^kvgP~;1$`xjJEk2GWhirvBLP=9IzrijHNVNjmxkssC2 zS&fT?Jn=)ikbVq8R8+z|`;sOE$7>a94Yro9vy4wwu2t?EIfKDhAcCl<+IpR^_1LAzY0Y#Nbu^nfO%Y zz)Cb)sYJG=_F#&hsg{U0Vj`5yvxHL>8|QDVrvq~|cT-BG%22$@V4Zs4EKdi{>@??N zndK3J*C>Xk!U>0JRhY}7Gbn>FKCH~@@ys|;9YRoRJTnN(BZOfS ztTGPuI`*$hhge9#Qr60HME1 z#AP)sk207X%>8!@RUNSoErOQD`yj=tH6^AGo9iEXd<5o3?BLw4^;izFFhv}q_Mv9rn?llmuwhP;aA8Z@t zDvJM(F3MMi(|AG%S&VFhcxXF1(+q-nW?Qr;(qIkdJus5#zL>ZW1=TVh$Ppo^8I5-; zA)VSkOeXP0uGl_&GQxCJHO3syAV`DHWiVyjkOu3J5-|`5rKl7ro%AMcm{x>KsXB8q z9yqg`;Q1`}6g@AYRwDoZ^g4v0ZPeKq^Q15FXI%fYq~b7b7#-t4w#1a=WBbV#2*Cqe zO7&;~v<+GS^`;VGsosS-@_6)34*Z!$0s6lPzmD*^82Ti_nfKyvDjuFJA`bq-qPV>-cBeWTC)gVNO^5A53e0IM{J;QD zAyEo`u(jlSgoAjj25n5ffhl3rnY^1khp;KnAV`Ne*k5W|xkPY$8fym(^ck{0(xJXq z_&tRFfgH7fLnZ7vl0iAti9Xa0{1t`XXMpFjXe%f1)&ehVEk4~u&nA7z!|Dt>*@ zm_i9SUa$_~G*VD^v>s8rAkS*41txfojZ$!~#g<@SsMf=$#~61n0iIdwxO~tA+etD| zf8>kvG;u}w*b4MrY8SIE^&jbg`p^tRqXPLPXK#0y$%u-cZpV;CbcJO^XrgET~Hq$6F46Iz0`nq~>q8rzQA zp(gB1qX1iuF!BrJi1Y}hoU$iQ<6Wc?*OFoIcVF8#}k)kEofqC*=qGWQ&j>wO^m3R|>))z>Ly0J3Tj^LP* zBxY4sM&0nbhyQEgKZd9aS_$#c&g6L&P5w8b-au~TCuld+1v!#bI@8q${T;8M_KN;S zThO>8??rE6et@)WR-ti&91+6C9@!o>#I|CJa>+*-l-Y(XLHrR9l@~!a2hU)$;6Fng zF}ezRvT0@B%u=Wi@~2eAKweA-(t@mt7^nxOl{G^O_92~#8)`=0N@dcXFp(ZP(Fh>> z(x^dc_#tZSEgCVX3;8Mb31YCGq~1nLGrvH7D1m$&aj8r#K}u{1@gZtVNhgv{Hb6?! zlC^-&h{^B>leLxLWGAFm?SVDakK|9}BghZG*oRC8=|Lq1W9Q2H5FSGCqhgR0ETaXm zL|(|S2wK%zD4%e#C4@n(WGzAql6?_^ZJ^dso~W2cR7ZV?ybzm>I+TOs4sl3F(hF%x zR#i-f$F#>Yepq{HWHD`6jmnux!TAZtD&Fa_QN_5ZGfxHUe-q+C{l@sxnPj8Bj01zP zdeWNNgKDetU?r6YLf8lC>7#NcOOo{|RehFpVK!9dAqML$gc4tdhZ0mtn8O+zXV_!t z4@6Hjl#*6zE#e?1%3&YkLS>dm4E(5gSZ2_Qs(%ym18M`N>{}U!^3-a~6SkUTWp*Zf zhR2|IM*WBm&mwAWRG5#xx?XXfMnm z4yHJZFlkgnG8n8XZGtCu@@@KKp1^UBTT7hX+EVo;>Ys;5>%Xs7eUlERmNcX|E^UfR>e`R z&orsphX1Bh<+D1qos4%?9tc(a9ZOYP!sr=4#9;Z#bp#_l$#P_0)&{IW9Qra2mASDz z!=zs=Od8hWhbSxORIU{W|EC?3#B@?+GtaNA1=i9*T$qIa#9o69vN~IO9^; zwhV*e6Q=5un4>z(QHl98LDUlF7*^#RUM~<%DRCiOgt8A=h|v>Ntw$Zy5~IO0qhlk9 zU@8VtQX4R3c2_YlNA;LPDXgSYVa@;kFkF?2IMCI!vev2(ASac#%0ZPtJgCLYhtXz? ziq*3+QPP!yaj$F#*ksmzt^L^Ub~qr!IKT@-sJK_g7XQ@IkXa+zV_S(VIk zmGS?Zs`BXgUli1KR>RH&Wx5kol~Wa7xmMMX;izZU8_fT(hWd+gn5R*IXXMI!o}KY7 zo$~a%MJ*GQ)v`0|PgM#@!g`jYJYteWwT{`k0=1ImRZ6l5)&KuZ$y!xy#w6ewYpT|w zZp@D>OHn-tTf=Co@*pUt$f0tHD2Nj4m_-mvm8No4b5zT4RGw9Jpc2&*E|Xum4Qih8 z!KVt?uQW256vmrj5nbg{W$p;4a@AaAD9aP)%A5$IN}xKX>Hp%$Fe=wEE2=GE9IBp? z25rxJg8HE{4D}%lsv`;&!g5s0#tL3NkPEA29N9=AEr^EUA&lum^_W&}0oLM&wT_kO zOf`g~_6L?3U1fX*V`s*L;j^XYmrj_B8Cu&TY z9JD6pslQYThRLjr6jfy-t_ovqU}we&>FD~we&aCPFig^gbf{XV){rd1Av%VS6e>ie zXE27xuyB+z8md#JRL)h!A^DY4#-Ai2C6h!JWA$`q)?k)b?Z#>dk69a`EKg@O{ogYy z8!L<-m6;Yu{ofJ@m*G?%H7ZPPm&&VIEc4`&3M3fvVgpOTj$h zGY(i{5QfRlBn?}{IOAE(6P%@tiukEoF$q;6|KCz8^Fdk6qD&5r54>+yuZ}7WZrF{)~XxC9}qFz-@zQDrjzh);P{Qe~YPlweFAomq)-z%%lqFQ#;r zQT4z&R!jd!;xDVpFe*_>bqu3&&41?+i~g;{I@G!{4r?XBs8-dUG*&67OnlIG$PZK1 zOQ{Wn$GnWStMU`Z%JM2(lHoGRtd8+zx)A(-rA#lZLn`_(D4lVfqBeLxi8yLY7&e|6 z4Z$d_Y#Rnq>zOSWCCOJMsF;=IqI^{v(}i#u2WDX=pPh+!Ra~`%Pp3VwW}z6y2>w@y z|EiHBtf6+JuB44xuFRFyktRfqTv+d7Uj0<&M*LLnh=qFN6KCQ>o`i7bQzVU*)H9A4 z)ZjmTo_ROY)-jHZmSmz_!ow8n5RUp2 zePs^RM#Qf=J`jiOLbR26Fiv!)8fHb}N0bbPXZ&Ct(hzSfSB5ff49@-~W>`eWut+N2 zm5`oTi=WEaB&{l>G7k9$^JVgm3MRlsts{xd-w2!hNX0?941+k39K=)kvl^nNdny^? z6y(FMUR=F^s~u2hs-bHb(l9y1iKSGdN~IjTP7)2`Q2Q~5AGO5V#rlNK%$`(>Fg(cP z2!=fHVBDAwA{OB>Z^Sb5bUd?l#F)eN^tfgKbF^y=_l(hd(kaJQ+^e{Z15(k!XsJZ& z_$y=4*g^>A7#4BF6d{zVwWu{|PBhq7q{o#$WDABtEyj}CcB)fr7;a@8vLB^nvHye+ zl#LU0BvSt&Hp}6e>Q$_&wbV~^|4ucPrBQDZH=F|ygQW?6WOE2&WCF9anksuZl2 z{F{vt+J8wqlIh*@l+vzD+&50|KF2fdaAfa8r}xjRZ>h)K&Fqc#_||mXC5{wqUnCpD zsz%rf{E!`(htOIL<`*h9qo+|=bwuE}!|_R98cWn(v^(a=dhE=sr`C{XW6z*O>aWUq zgpyYx59$GgRWzU?K^mlI9>V;B`3u7FOFl_nfPC1P!W!a>5b`Uc#kFaS8hPNlIktk7 z-R5YAR=FpH=ZB#&u^G_*pA%|l2{{YfpM z(TREd;@TRV6>wEE+w+gUk9(DI*D~(Cr#<+%iy31ypkL$uWgIi?8iTVxYEE84J&D%@ zCt%@$yPxrk`=W98Jsy~&9sVeT_|Y|myaMfl?WT40nBr=CS}~73q*{$~RV{-hvMq8! zjnG1LABQRRC234Gh((-`4!we8v%S>VgCy0|$%-g7HX#;ENGV0-)qufk7P6cW8ArwVr&Cd=F<0c-;YD zVDIBE1&p47Q57h%2E|vv9%l0!S{&c>j(td1d(@D8AMc@1L;RA*V!Kc?(hGINydOyP zgB(T<39UFV?hC$yzIw%h@2khwU_28%BH^GMMrWZCgW(x#5g(~Aj!Qs=UCAP73A7O& zv}+yPOXG_6SKy8f9KmQ`>U(Sv+MIpqnt)dTnxD}Yv=<+vQ{Y_AW^g=H6bn4#UVew`X{tD2YtcYUp_y5IXz}OEMg9hU>Q0xed8$mG;FwO+Zr8oz8z6i1~ z#sNm6!1xpxdx9b(V9W`OEpZr9r*Wy&;gp$VfPzBVil|0cV`GaNfd|{TNdw3v7lq z#|T9a;ETSFy^ql|o>xT3cny}th)Km@Cmeg2;#Cv%$5r##8l17!D=f|CGzXh2+C#Gp zrZE)}h%gFKQ-~&nM?-+rfyhI(D!wtGDbA!cQ{a3ORG~XYJ&J?4NR5HAA;e&U_g?~s z#ueYj6;OZf%@W&Rkfe31CAw>$rZB=n@2K89e2570(NJDhZ0FD|_Z5xo<1rC(k z1GGW?-l@Irbc_*Xas;p}w< za@Q*6%8TIrk1FO&90xcO&Q;9y7^&{%fMcAl%=E+p_q?G0eJqGO4e{q%L#y{gG`_ci3lFdcplfd~o;waP#F~PS zErGaq7KjSCh5HqvxHaXQix-7JK8ahzr|}#4JN#aupZJBCAhZ(xglLd;AVTgO;XMBW zB4|SVU3f!zL(U*H7q_E5Jhn}Z-ISSJ(VC3E>dA;#r( zh-T=9NUHPTZ7mNWzTkdtD1VHP6heg*Zm<$BJCz8q(>iV#M4jy+U6ooxziyZ7dP-ae z+~*`7Z1kyTpsUCk%6PL4chH}?g1kb9x4k2_r|R*VpNc?dsMxGC1tp46St>kR?=R9$sL zP2(uzX2TZ!CVi6rfPTAf5WFYp3-PMZL(ucfm3HzXX|(5WS1adIM~QudO|jIq+$cX% zmR?f7q^P)B@!`Vmg)NH8is}`eE7(`Cv#?$9kkVhv1$gV+1b7$uGq+U^_Wa@u@x+Ld z;S1v_hylLI+}WqK|KEXIf*u7<4S4APMZmGZ;{o6JjrASk{g%0gaRtOo_tm`?CM&Yk zQ(i4~^nB-9?MSl~*jhQ_oW+g+I7T|-+=cE|(rx7zzP9jG5VZ>pUzx9Z4fOKzKIJ

69^Wu==6-BcO+Z1jnXqmq-&y{D*H|A&MWMqAtm7VR$*_sodeIk264p-2z zba2_U!Ueg@vXbAW{oMuG~V>S&jSA=0Rw`PLPEl>g!hZkhR>;XH>62O zROnBkivmTjlg1cxg1L*~BD}NUu(Vei>3$0B+Gd+1z$J&9+ZsorT~+MV>SIW_XzW?xKqeGoYURiEmO)q_3YJeHk~v!^8POH$Iv;|Lc$W@-I}#x6QVz?ekHOXEGt9|ITqY5 z;HuYA{Z%cm{aKhPf9P83h_Jn5i7fx4)T_i<)UxD@5_geUw63Ui$=Kqr3%P>t@(1UB zm$fJ3T*l+n(@D!;-AoEio(0qR#|3`5VJS_L((`kb&GND03k7$j8a}+w1GCNB84?rT zx|$quyvE~N(Qj>!eg4*y$Ppp4L%)o;SM4Y7#k@^k!5xwl9X&1UO8?56k@ZWqGk0a) z(41Sj(+c|*Hz@Wlo#2YrNVNbs=Y?ev9cBQzs@PHmyddxMp>}tTp&8DKaPQ zkFd63uE1#X*SZkx4NX5`s=J%5kImcB$llI!we(3zpYr{-8&>Q!zT9c+;`qYSy?9pP zfa2xF-3z8?pGlvQs(bS&amMrSpFMjy?9H>x!@0LI>m_%4eWU0JKU&k-)z5NDKJPa! z+$a2D=&i8&QJ=@^Ya40>zqK*;?HXT1JAy9ySbb*s|6qPsHd|d~pIXk8&MEja`;-I{L(h6SXK8sC|o-IgN9KY+?!mQ!NOH&%&*FL-LFa*@rr<5jG zFL=L*8WEWjG4ZWIwcoBayXMKb5w(ZK42kYtqgwPRuUXP#rLD1s_E7n-tUejK?6z61 z^oD6uQg)>%=^1&W;SCGREeqw5W)r;s_;QdT;z7+H8ZBzEr`f2+d5tHw=-Kjm!)>)j z#zw|W33vE5HFOh)N%O69N_Lc#7M;(@%E-<7Ex%#Gy!^XG#ih~Z*=4_5{&sA)ueSEJ zu5pU)&~kh3;jHc1Nf~J`etXdO-t$Kdo}@i#lr%bTKwjyKt@n?l&oF0(UDe;R^i=vq zG^o?D_HQ-j)%~r(!nn9PBkCo`N!5>p@2P%0^pNN*9aBC<`N@4dr~k``FY3S9p7L4B zxilg3enzM4WhKR~i4xC!?lm}Se68Hrc5kg}IJezbop-j)YJRZkvnC~tuGHOC<95Wd zz~0{3+VgTtXRft-nJa%z)|vG9wE0=b@_#D+y+|xrSU9V!#2)GzYHw++Yu#iIv9~On zncF#aMDp%e$DX&3pM7u0?NN8zy^Jq7U~Q3|`Rx7I`=!6aPKEJa&kUg}t0yH>}@Cw@208+>m27wH<7ifMB*))%H0u1sH>H2k$OqcHnXjQIpLo z_!{j3m-&Ad6;Y#hR7&XWkY9Wj@+%<%{XgPlWr$@%L2#jWQCQx|oG;7jd%l)lyPGKU zgvGj8-}l3B#X4#Zt8=~n!KRyAH*8(MdG{vI8&0o%BC12USG7ig$IQ3Hvx-^z+wN1C zo%6ithmw!8o55SP^D`EtHcp)P{K!k;W!J~mA8e1`|M=LgA1<`G5PJ3EjU!Kj%i4Q? z>Z`HZGV9q+_`4(012=>P*7mJaFKm|iL!Z|nbE{vB9v8VcFjuqIU02zm`&8G^ecC?A zeb{xh{BeP;_?)eia$3{H@V)6BuWr6S1wD@_kL^*<({xztADb_4aI@}$I#=slthpj~ zd*r^*1OB;2i}sCpP>FMfmyIkwQusKxRob7)yW!p6U%g(HV1GO);ftrAJgj?f>Ae$o z58QfnW%I>{=a*l|x&1JErJ<(xFx%mrDb_ir4gM+q?ITm_{oZ7LjUxf2UJnB9hqa3A z5VJmNdEgY?9>J+;tbb2mS0{?Iq`i*U<;L;^>pc%y9 zHZwFFTia4Ax#pu7zbKyw@5szBgYOrbY|l=3(~`~h%+|E@XntYFnAB!(j;Cx*ua@fj zYQ&Sz;^X5#iZ8eqaeMMz{=wG!ZErWYw&d#Zdy6xRB`;$fzu0!o-jw^!c*-j^xPPtR z>;G6ICfMi~7)52}5q?#x*`{66;$5A+Y1V;msMw)15J%Z1{&yri`1 z$p>G3ntUwj(6jLgeUk%I42TSmiVMpNx*E_sG%;d&c&o^U)o(@zM*duFKu|*9_o2EdA^d5; zR-eaSwY)-&3%uU<8y0dtG(5y7WN=9Bz((F5`@b7GGK816#TObgAce18mn9acAbd8{dVX2i2d-|9D` zmPGZBI1~O=bhe;40pg&?B5liDoQJ8ljctAko&k~a_Qdup_yY-B3^e+ zy_a$<-7Du<7MC5Ay)0vKde_v`8G-o`d9RBam29NnVHwyCS` zPhA^xCsQxoyUss7f65uIexi@QnQ5@F%(}XyG!NdpsJ-iu>@iNW^0nuf=Tvb>K@4}+ zEjqjLyPV%{lBu5!Q7BzbuU{A^sS`BHizSJ!9gM*FSy|Jmz-UqgKYym#eC?>Zq5 zO&86dd$sl|G(GoU5L6U;)4Yc3V$hjpdkfw}wAo^PucKOcD$(T)hu@7|~R zokB}~abaw^#hO^|C2O4{b$&r@435Hp#F<4!&J=Ag<7iI}uV>t^*4f1mOKW=u$b(GD z#;fArWiRqF?5~_lr5f%fQavFK-loytk!0I!nWpp#Iqu1_28zoqr|nOi+42eB_yCRZ zj72OT;;v?YOCD+%=kv2qFPFyg75{5^|@ zV6Jh4{;~3Qd8(@uydAceKc{PNxF+ni4|Hs|+|*om=j%2ZryI%*dT0N#29{81o_UgC zE>}a**=7loJWJij%q=zH`t!x>?T1YF9d(SCO$UuX>lc*gIVQ=`ntJ|rWxQph@-@7> zVX|YLsL@w9j8=MClWft-3a=P`iIi(tEE>ueIjXy#=vHew@Q*Z&MF$_ATh}sLGeg(Y zCtP>Soohes$gs|EuJzYz-0pwOi{X7k36A%;{qBQ#XCfo@Q>?cGpBF8H-@rc?opm|1 zsmgJ9-&vZohiO^zv8nzm*>>ZlKHX9`8Hc@~!mJQO|Ht>})yanjLKA6=w(G zCU2L#oddY+(lfp#@V>(t$|BcCrlYp=B|%&Vzp)0Zq8DpOKCapJ2J#L6yMa$Ek;dN~ z{at>7qj0@$ka)s3F6^_ynwD)swx@>iSBH;ll6Q{MGH`_LZ0T11w*Gf+cgbG`>b}<9a2?cr=o+US;99u6{DiW; zN?o%}yaw;4$Z=oue5YJ7z9Xh89E^cMh5_7mT|<+nY?7yo;h3eDeu#UY+|g8DIV_zm z57JNe4G_B6oaIZse}m@@hka^$`5AAyf5>^))7I;aai<_Sw{y|1?c#KJ@^fEsn*R2x zWBbUtM9~#xxsGbfye{#hTt}RxLcV#bG{&*c^2WGI`$9BI4Qwu-NBml8Io}7K3E!}G z&_D3=@z|V)jef2b4h^5Io93D7mFwJA)W!RXeTZogyoGqU?v_y9{evgF@Rs{=pw6^H zud}+Hi;Uk&)16+PUP83c(s)*=BbY4b^K1A{b@cIyRqDEOLpfWz?WlCk-dpVLK4$nP zAlv<+d!zaL;w)=@WwP589INa$Y&5(t^%wRPF3aj};C+Ue4{*ndd-=3BH?uY`4J|Lw zoi?Tz4mk@;yNC8tRur$Z3{*Z38*aa7%?Q{p1?&E>T`kxzwYE?3?rfN&?D3sh6zO*Q z1QhMHFNgP^wAGYyM?9C|ou&)r5K~I|So=jyl=G0u>@!LeSeC9?pzk2ec26j~1Mg;i zD3+8Q(>xA3XFFcTOOH+S?6EG3&jRV0+`xI&5ov7cTB{%J%C^1j^R|-a{+#>3A@MOL zeLiP!@Qw9by-#MX)g0jj^D$Y-x+5>wh5A|z`-7v~F&A-9I6hGw68uQ1Fu+jpb)yJd&V&sv*0Ek57cJ~N#5 z9}Vw#`ZixMq&W6C?ui53$>z_3S2->f&$eAI^7i+2H*|}7PM$2?2$U>iT#qz@rGx&t zqo;A7oaNWKuuPa`jBso8KGu4UANU2nTaELzr&C9H&E=v?uFHwKBJ(!iRZ89BvGz32 z2i{#ArP^ZwN4U+NJbfLZGaqkvmNzfFudVAdPoF4E_oRCHlyxm^sl4mjtli~3p!gU5 zBl8*E$HsTd=9jN^FBQMn-}5Xs&C>7iY~&vm-Y#z~rAsbjXXD4lTXHx&^IPYrC`=PB zaAtj+;gqb^-iD{Lvn0jRMfh7tbhLLgmYecTb&}_g(hatD@b>m1@dwRo_hxv{?Q&-> zU!+NqBXoQDNqn@%V=XWUvbjXd2TRSA^Sn!bUsIyn=9pe~-nm(91MfMVCLPth(J1^0 zA(Gqd{MfO9`_;8daJup}kIWbO81aR34F8dPxBXqsZC4Y=b$O_6HrGUWAx10v-Jd#l zcy`0v6D2WPUo5$lmE2=^8v0Z^tqhXZN}n3`bB8^@=mL2ce@0vf?{c{;U69ua?YXA# zhOgs%Q~8B5T)ZHE;pwAHlQP}U-ID!~?QNy2vdI-Kedn|&-@3Y4KDM@%2HWphvOHH@ z7o;Qd26*SxGI-xeJ$HhV25%BsDE8FN*R~NQ-FI?^cn03;X4bana`{_c-x`l<{?aei zJ{6y7Em}p>&ab1PnX#sKsrHq2thPVQaCJ0c2HtC&Zkskr{8V1(n&?_+_i;6I|KptE zTI!tRkZotJL(1MR`K5SC!Nj6ng$;`y7hEemThO#PKS#{ynx%VFm{pb$oROS4At$!{ zOZzhGYxf0incm_%!F|&jhpxsS~t3aFwsw*U!h`wb3|F6R+zoUgny6 zVr+d&M&+JQU!Sx#A@iQkt@@X0U#NZ}@<7Didk2cn*15Lpfp_Au`~$)dfs+H52HcEY z6=#c?71O?UT*I%L{LrYR?gzEI)eeqP0v4Kwi({P~EyJy|?XlM1%I7*ZxMZm=9{}$X zT;dfJS|h4o?V(LZxBR-rycXSBMmFFdfZ>6`a?%A&-* zPg8E6zjFAW@_(cgOO6!oz5cuRAF+o5&XwIiotkV}>(jXQ?&ki@e{8v<-PbL?shjcE zmDp7^46z%cn!t0fA49u%X}MnZA%(Zz?0nhp#ZPa9%qv-+mwxAYU-y}Lgzw9+B{B7C z_pYDVROq~?`^>JX-MH=zI_sK-#e|1n_GzIXDbKfVu@#mtE6FK%KWA3D>(viWo%j8( zojMUufBWTCi{uyp6-L7?cvvJ%xf^PML^r& zwtL&p=<-hY=Wly;9M`T>i=A<~k-zx)=><^L2r@($;o${myCmy!1-FE$;tx zWB-NyCl{PpcX<5%di&lx^y9@t3CTHcTp>j#+%bgh*;_Ag6vvc$w~@uwb1@flBtyv{7T%uVzi7xhQIA+4`;o7vgYp-1O0 zx|lkA*)+LPa-*R7<7z&yKB`)xpGJ4q6s(vQur~Rj21&%_Glsju=#<@0eas-ubLsT%+Z|xyEOPe{{2Xo%Sb_ zpYL7$1HRB5RND1*w+Ab3|9ZXW&7j*0pYJI)>kF&RtsU38M$e^vjlITo8QU?VUH9g@ z>aMSuRx|gl!_hxQHjL;Wm|)xjcdxTcZ)RFw|C+SqjgXR(JnmJuuCk9gU+&(KLDstt=Dn%4KPW*>ze*R9~Xx;mD5m+of{U*5X+=vL9qC-=|1sGq-vuM={$p02}3y)O-z(XX`E%I*W&U2o{FF)^lV z#DS1{p*MrC1&;R)Fn%SraP%*EFY8&#?3AKpXHtA}pHx@6Id#X&H4h4}CY&nXSG4Wv z_Ql6e-!dnjF8e8BeD|)y@4Yv6=#vq%-;Hf{At0OY?YUXjGV4;#Xv;8o!x67ckS5vl zb6>p-eH3+P=FPshK1eVWTMcK!Cf6I(^~HN5hTrUSs_WaGuC@52rY`Uo?tXQXz^E-va}JK*xTezxr|a%c2P zbi}W_m;W#)@sIq`O8r1b-3gsO?_Xe^X)a31-&xN(YQ1*x_;Vq`X2B; zW5N82@8OzI9$d6I=ex{D8SOGV<-V7nkei+U{>$o*{I13y9`NhXEt~#qcy8_8{EWkX z?|0fTboxl=m=7k&qovL{QD?Q0wtvzSpJqO~4awx9l=4H)1MbPzTLt0i#mS4Gp&vL6=%NzAS&IyrIh^er!DKv)AX5kC(9{ zXLocdP09NqYj{?t>~FKb%j}UF_sTb3xfF4@&yGK~uRHM5xhMCh=b1vk>iYfgF=J*- zXg8S~wYJ07&`&*ac{83rz2S3p{r$a3)pI{9bGqE{ezMgLufk@j~x6S^^0|Tr9gTa$~PMEN0M!}q)=K4(B-*sBZm~zvL-j~u3)Z6bo zZoJhvF`=-dT-$idv`Jqs2U*;CKji;j-j@5o_jt|PomvkbIr8rC83TuRyIvp0a%sai z1D?TLoiH$Cfg{BGgGeRLzrly~j=r@m=tH4tS;w5<)LMzJpJ%^ZmlF2o!OQT6;*InZ z&Rt*p8h%i?x+F2yCC6PF^7D+I^ZY*TH*?4PotmB!yk57wUjMHed#>!C^iR_#HF6VN z_jKdD>-lywB)P5@Z7DcdcG>0B?x;4rx!kAGuvqjJl7~L)b-&q-(WfWxIkD~L z{_R&zB|fu#UgPrMZZl8LYd!DFS-GPe%?D~nJuUirM-_`GfEV_TG; z`K>+muQwUeq$uvE&>otcQg=pn^5-uGCYWCwdmWm3FzN57yKYze$F%?O){8slocZwS2Yfs4^Jdi2Ge} ztF&uxJUJ(=r-dB>&FbCjE)Dx@M9iSZy>_-2YL3xuDN0Fmy`K2yWaiUyLDwZDDt2p} zJ}x8r)1b`;#aXN1vvfm>m^>}XkzA1cZKC~Y*_}=29vwQr^ShnNCq8^KxM+#jpmvu( zYCgZu+=DY#Pxz$IxY%K~L5XW0+_)BbZt7*_>5Z%|rJvcCI{oc_<)0Ne(w--mq^~Gn zsf!H@Zjjn#`4IQWp6{7^Y;WEtCd)Xg^K0 zaP=s?p0_=xQ|`)~b6Gr$?yQ7=ZfBm|c<`&={kE^!-{$7S+(Cx$dh*~&vyRVApAtB# z?>h-~+iG`brahe=|L{@AC%H+{`S-1lT&ErTN^QCKGd8|1fB9{)cfl?BTmL3Cziabe zpU&^)4IbJ%viwuwcZ!r1>IiW;rW!$H9Yk$_wHqWXqcv7u30U8X??$}X4#{XPYaf2ze&$a zTa$9`h3o!@H@}UQEdG{c=lAZzR!4y;r)Jjwqv|W*qqw@aN7i?<*Jh&GNw0Loc;w@I( z3Iz(KP^6TiMGB>)xVr>`2MGj{kPvrW-yQkR$?bf+|I9D5<8$TQbDn$doafHZo1QZ& zuTkaq+

  • etgjFdIK7EYY<<5ef`~0zlVnS5BEzCIPd?v>6W*jX|(aJzJc1to~5T! z(}fN0J@D!7PtIJ&PoPzze%ptfh8&YdBfLH zLfr9Ad>hZ^*|FoJ9vY9zzoBEOYJREvfivBi?tbJ||cXbT+K3yP(a~PLtb+ zO~=Nx4-Yo3RIKDG9gCdnTnVln?$!KA&f?ITV=EVxJuZ$d8dM%?OXQ}r6&|ns(?b3U zQ-!|?+Ymk@JRvyOzjffXkl%tD`*!huVtDO2OxsJ6+H0 zRh7Mqr{-0B8jzWnUh{!THN8>1nD=BU*hTtr%a2WK>Ii3apCa3|-q`k4^Di1+h`bc| zo#D0I4L+^9>`B>chG8$Wq0|&x>++5zgA1cz#5t#IqhlsJNjuTg*XN(Whe5f%i+ra0 zj`X|eKP@mecx=du@Q9G5zCU;$H(XOcq07aG@O@KHu?@`jdP!DSk4%SQSIt?I@EGy?1Gr_;f;;Q z6@Lk@`Bh8`YZmOT#qPu0Tw=4hFDX34H_WJJ#@D5c*v`C}eqO0hX^^i|xY|Ly`7tIUVGXA;yd-n)? zo66(GTMMq_rerlrAC-FW&FtiaXDyzLOsr0vms(ySx2Fhu0zNc9)pBa%@YofNde({b zP1nqX?{CuS)_e=_Vc!(SIRCQzUY%0*LrIsiHf6)g!|WSm4>T|gZWodO#Cbmp7UX#rJPA^bQRBjYG z`wM%%JJeZP6>PT z@@O&EI-<&4lVH(Q53k6rcwG@)I<;VBc2FkufqVJ#$>gLPcaPmIe0RpIp^vIlBg;E! zx^8cNAo@*+(fgz9mSs}0P&&sx$o1Ox!r7a-D_>31u7=jG_PtyfchWVS8RI4Toiw%g z8tt>u*I&=kU-Jr>`*j)L+x3;OQ<13CcpTG}cr^4X@p|XkQa4q1NuQx_si)~TSN1mny+g1RGCYRMYD?*7mN8{W~F^No*I$zEUDwo zwvU$SSZ#86W|FbRR0UeZ09aN6;$rtx( zUY^Q*T^3`TqD=Pl3q2fo-+Kp)7B$drkjtqK^a$k)We&x-`Z#;L!tG<=ON}{HoRU*I z)hf?A-dpwOl+zgnwMnQeJP?Y+zu4Wn_dYxQ@AxGKEei7pYGe3QO)GaZ@3^Io6Se@y zRM%|R9Cx<3oQ;ODf*0yoRU1X7d>>+yw2_S?#sfSRbp2Gb+2 z`zt*ER9&E7y9V3uI5+b9#qWi??iBY$zC@_u@`S#!P}MZ>w0LQkYZ%pP*>jkKIf3rS zUSkyUK=m!%JdX^0f?>DuS5rT4+S9Ge)w}~KwzvklPTMRzU%s=zoxbJmu{XZSiGcU> ztaCjRLV_9&?x^WGq0Ri@zhwI?YjUDq%)Zy*5&J%@ILFCrso=r&3mb>U%n5sFT&g)P zepjO``lhgL*^$a$s_)s~GWT`8z%%rhVX;?@)+O6c&4yX&_fa>fmFzuLU9CkuOf_9O zNcp|uq5QNWKr=)C$a9o#x$3aeTYg5o$B9{IiksD(vF#KedTt8t7b*{!Y-(-zO0mM$znIV8S=6a?V^yT^xlZ$u%;=U@lVJ_x-Vi~oZZX)~+9^k3g09$?Y?5Y#h z>6XSet?jk#u4A2RkE?-wR@K!Ks^sqyc}dICteQ>S7^b6qnL5Rzway|-6!U~k_n-De ztJBfMJ;(hO_m=LeJgL^|Or9Fg`nn6MTXJva4DZA3;`@p=;jOrXNtDf%O<`u!-_enj z9lk0(Cwxo2p`Xzom@)FZ$|UuEZL)5rhp)#k+7R_6#XGhRT_aY&%;)=r-=Ma?R1~Ye z)XwyX*Bf*?b&9-(9>W8YcCK-S^4G)+VVkgz znRL1zb%o#VZs~Tqx4YN7AG?;pSD~zPx8q+&efI^;QmZ46yeVKi=y$?(l zdD~^+f8ka-ui5rn^Ua-W4x4|l*X6F#A+j`?Lmn^Br_Q;5a$a)ob^h(#=UyXTfv;8* z>BWo(+lirA5BVYaDX_Ag6~D_988gh|9muqx_rSNzGI}I^jamjX)1RO^Fx}Witb=VL zr(vuxTD?iVQPW!YNcT?DKy^_b$IcUHxjDxW$1vw??lv8(Isx`7s;_vRd&M)mf?UAmwPzOoka_ShfmJMab!JOa8 z%vLsrnFC)K?-i<@I_usVwYk4_zGD*Spz6ubE7b}=2I`C->uT@Fv5#|xx-;A>xwjyB zZ@8VVna*4A?Wo1woAc(?;;%3|I*wCw%eim3xlpp-aAU<}>H}3o&7hw$ESpb_0l99+ zO?6*&=Gk{ST5vnXe$083f;wH`Oe+a9~RQQhVBG}m-N9%0(A zRD0wjWH4gNY=G~e4~t2{5cm>%JH1L$Q`3OZH#Tpo?&ZZN@Ui?(oHb=grx9fphBb=u@%GJtsig>wNwusj910i5zies{K zi2IEY#`w!t$OkDV%d;7)xSAj0UgUH;Cb*us+i|P8ZM=og<|cDa{suLOiG(k@6Pfk2 zTzKvp=IjP?@Y;2S-%4L#2eTQ}BRfF5;;5fMYd?3(;0>3=0y=>$mYt9{kr%VSF^}m_)LC%=e5d{^eE&Qh(ibD& zChH7RreH46UUWlx1*4R$lU-nASzp%1Ol1n_3v?4Ejkye?@ohy5jLtmbxA8i`Dz*k> zt7jfjCn0*`h!Dmf<%hxC(sQAWx`l?~ULlR2#5WTfQ3~cZ)1BE&=THY>^mMDMhr2O9 zSU4agiY`d;TUtR?2tNzW`5o?N?(uv*Y5_ftzQR^CQDW(uas-#tJoLxbgED|$X9W9pf$GP>TnzQ zQc=MSXE)3C$QR51WP8$QVO01GAIC4?{}47oMCosoKs}(UsgHCn(~d1-RxvNYUvJ=t zyE9xbT^82_ZYsQ^Le^99QSp^RWM5Oa`Fid#&PR^Z&LFO(IFa#`f1${b-^l6ALF~AJ% zC#k9MRk#NmNKX*Ggw5Piw7*mKzk^o<8*UI3o!{7#h${q zyq@33-xT7g_w;b~06UlYg(?=d@Ey35t1Pd zdRnBf(}8pmgbN;k@){y`hiHH;kOvCvxtwkQy67rIRGb01#7HxYdb$a*7~4dLFdW{{ zQ#?dvfh7IGzGo+}VN4H@-pBkwzLNh!Xe(w?-q04)=zH{Gc!MYPUhD)>Ij6+U!0Pfd zwVm2XU88*HKA?AJLwvc}#3lU9uYi`W1(N`+OQ8DFBj}Eh`?)X^(g}F#Hjv!skoKe06R>nfx;3g=!r(u35e107n_P}AYREA;=aaF^C|cW1?qb# zq$Uz#t?UqylnBvCNf4D33Xy9|so$w%Py%g14&FoT%y@AEyx$kz@;AiGP;@SJ8QyvU zN~jy;B^Rz3B(@YsL3Ak%GW3dS1@E2%a_@vz5DxXTOI!eN-vRAn50qgaKmyw#&3CAa z@I(d3_&RYa)b3)4sS2eQLwyudC2)2Q#Av;T`O0@e`4vM{)Kbt}$Kahq;J61wRN)*) zQ4q~G7)q2x%XNatuCL&_LvZB|sEdgZDOU>fmal_%--MQpPfv$(^Mx|KN{xY9Um}i% z|FckcZ6W68Z)yvaOJmTiPG}vapd$>xJ@XjO<4ov(!82;aNRU*(3h<`p(Bd;7)^-VK zm3SzDWzb^$AX;bx)X#Q^(<%imGggci)nXt-Ky89tq(bg35DA76_ZT~033723>U5bn z8zQUvLpkq&8cc;4G>qte3pJ7eZ^k*(6KdywKML!UP*WC2MK_2y>k zNa8{$#VPPx4{G-y$m(eEJ9zp*NLesQMRSP#qu`0xAT?*;c?A&ZiSc>akhU{$)k(;M zC)COisDX))N*9#eVW|7PFb0zba@3_ZLb3});1$)L*;*?ZXW{PmP$tN6fV08ap$<}O zqfV0`3NIC&j%8L1_4o$j(Jq0sU}W-hD5q?=6611lhAWJdMaG~~j7o#&6~WVdK~CDh zTibxNd%}}3LL51_JfMUyHeC*-m;sS@&mkf#5AxyxcVJYgC*%_2S}`J4iq9qSh{$Px z(dIZ`x*V>fkBlDjWV^!c}LXg&c%-ate;$hJB0z#|5LxF>3fFly4F| z;SA(&FTD3pkduqB{}7HphZJCK-UgYvR4eiK;BDEEa|%ia=Yp#N9rY6K$0t68vZ$Q6Oz7m&I@%r9lDj#M(!$1H1-f zX|<3(7swH&40#c~LG~j+IznsDqNL%~BqCLc!^QY)%pZwdB}Xw9oWyBs;FrYN;(Zv^ z?OmG|E4%}v(`z7iI3p$Dk3c36WR4(bkOu+(u{~itE#Y~x!*)583r2NgDPoQ=ZXM&; z39m#coIxoh^@S3McO(BqDctcM>IWqb6&M<|fEn~{Cbs>ilmMJo|5H5$B+8U5U&M-om@I_$6ImWtUnGhZeY}w(^ChLQ3 zz=&xQ>x}W$Xe+QxNdz|@!MN)tu*R5kjK0RG?l`yu@ zm@tJ989=Q@qNhnsVC@sB!82qJGyFqT_ly1;E@xj^|wDMsD{TqCQ;3&;i0uBby%$0F;; zTlmF1ror+a$^(}SD4k5$!xn&T0M|tF^PuDi=SVJ;8kQZlu%gXAza+X?OweiO7wgx!QNNFetD{-b@u8YQeKSmSu?e-{97ptJA@Zs9_%!s}292y+8+ zHy}R|o+Dgv$S#2kG6118U>M)zYzC1zQunBz35yA0B-9%OBca4e5`#8Y(kv)*$O(cHiZX?Ehe#Ak z5k8afuHi4Hi}0vOwu-Q+kYiFB3F8##ZCvoJL@!}Fh+HEl1rG-`>`MjidjCTKXO-M)p>l|xZs&7O_#8MN^G^}Hk09+G&1nTBL znIL)xk4hQ{&*7iMl!q1y|F920ju9pO{0EOVn3H=I{tXJScmpbMYMcC2W8aa}bslYB_9MM8jhlBf`O7NzTv)NJxlm z6NE)1khEH2F$ikHIzy>Mxg`HY58yF8L!_2i7rYAXE@}Z{gYXxxz~kh4vL@G~91tCW zEdrTHBpre2M$1lgHPIAOJ_*Yka+aVpkn)jA1bb45If=GFi-4^c*Vw*EsgwFfdxEt^ z)`(3BUL%}71h)_rLpWs!_Zafg5VoQSz;npSL-=@*6$sf}8o?TW2`>-XMofkXil9Bh zrGqF>;>ba4M$i{QgVG*BinvF3ad3@T4KW>h_K=AOxpff7A!_Rg|I#r;g#;gxEAT2j zOP+vl!$tbVCu8ae(jyp?AUkCAA^bv!MaiC&N`epZEOH$oXAtI`Y~dc}8~5=#d~n#HUEg6kZiGkQ%>Llc`ztmx-@ z#nQvN!7@gSK7X)R%1#9z2b_zSTZA{>Hq zaE(0yyqY|LpdCCSaRd^qjC;g-{;#DZJrA^eXa&)4NIVjVe=tXgeQ+WE97L89MwUo(Rq$P|iNziBBQ#j1lpI8+{ZMbWMf+( z=#Ah-(n}$j53w3SZ$yTrJ%Zm5xqmJ#EG2?ONq-y5PVy~a2@_OHA*w?@m;V(%W>3@(^<8h*oP?m5Za)oE{A8o{E3xP6*SD+mrHU@hFcnvv2d|!AC zx$g5dc!XR-?j&y|X&}}P--l_(JfOD~`#;h=4TQOttP%HOt5HGwLBBTfTVl&1zFNs< zVB5oXNLn+tV2QDk*nDCM2tq(_ATGpDi6{)chXmCk=0)U5SR@HPWF|aipwnx?e;`Ye@wnoCaQSd*mcERnD|NZ0rcn|)QWB8P>YmeZ2hSa`) z;D71pSDTLhwa4)cp8vc^*W)kdfc#>bF*p5d^NRePU%?ixF?YBiVnh^4FeUyXTEvp- z4v3H7QDpc;j!y~kN!XSkU_^Mt*G!NxB49-Si1~0Wp>yQMjDTYVU*k5~eO!}zCn$j6 z0fH0o3_*g#+M_-vTAt|q&sYQP2*Dl5+DR+~SxcxPsP-s62Ny&Y$kB;sCB%Uk1Ji)# z@E^AcW|6MIV^SX?yf!zuCW~~2{Nh!lNAWq2m;#atOo3#putlI9L)%FBqKS>deF@uQ z%R&T$))qaRh~EfSCAtQ63$}9!0iq2jnh)DF>ITUZKs+JDyF|RTlJ^XCt>pD49yZCV zgnYw@EYKH>njGzs)E7sjLTmuh`)D~NKcVE8#4#}9XF`vpG=fL`Ly|uTeMf|w8a?0W zQ9|E1@s6Y496eCzJx8xP;fen2IVarF#KTUwr147JC;#YuM^8MC!r>9}i)*}7^3CJb z7vUUvF20|<7q^JF9{upd>xG_o%oFB@FsBmVD)FZhYfA7QuCau0ERc9jaXb_4Fj=td z&_YU{_0N3IWTXjef_S(PDG=-?Sw0EZp|3__U?!eVf=!PHCknpkyGrc&7x zFC(^b(!vp85ML8Prlg1a+3up>i7;%Vu0UH%xc^CCi1dWe3L`?-g0_)*NqB^49mK;D zHpY=#vWG`;!6Ss@p70@)c?1ZOmQXc$0%|g%&4{fh_#2-r@#16ZFfEuuX^W(q^w0?_ zKFP1tTPNc=#G{HR3w0nF*&#hSBD(~&U`b**elA~9-UJ~L=S${c_)cqlK3$ZuVP;d(YmytPDI^^+7Vkf;$VW!Q77U)(E$W06D@&=n&553%F+lO z@q^Jj3Xz&b*_{n@)tBpfXoLEN<) zwe}8ep5&<^^GTpo5buT5E+o$eX(_nIr;x88$dicIoai|sEyO26Jm|!uPI>@%mW()( z>(GwkRRqD47D}`q*(SYZ+(vzdXE4V^r;%f*0}0M1h#s{bLFS(|0NKaua6#=rG-lO* zPegAt$}*lM?5&kAeQaHb%!!v8Ti)juh;7pVsD!k1fE-!3A0^sy2wiQgz7 zqC|f;euF^1b3pGR`QiZmkLW)oK1s)@JvH-L5sZ585D$?rt02C;8wZAdQw?F)XB zh*kx!AS3r!##qY4LK4e^Yl62(F9Yog8L35FORNdnW86aYglpn0lTa4g8*&GtJK`H6 z`vmC``$WD9z-#eoC=a+zEEZxh$uo!VM5#ffiAPaC;~qh5M1F|(37<>)Mx@6i`S39P zW*EikdUSYCJ!=>^~#7vlHBtB7Bi9FuASJ#1v0 zUph<1QpuWN60Ak?K1@eiZJH!}OgxXo*Y+9xNM%XNncyQV6Ra7m4{USNI5C!>Wa&vS z11+Is@v(K|g4PM!F(fO#)5&7dW{6#EIumI8HlDa&sXDjsif4f0DWmhLNJA7Ji2jh+FI784si(?1I84Rab1FQ$f!O^7d}mr zG5ijbv>?)kupJTK7q$dMaae*xRtXM4c_U~J%TFphl0wWWwhNSc($26BPt7E4 zt7?Cj!}el`d%(ztnQ+f2@cH%u32Fl6;R~(K0=DTbSoRz6E&Dcz8(9Le4+F$@Vtts6 z$1S7_H-#e*e>g`NDs&cF2r)va;3+5t6-0H{6WYKWH}fEF_`2{$D1rE@))03y2qKcc zgRkkogZPXI5Sug=)}tXRWGp;)F~r2|t&QhMgB-Kadh3H^V$8(1koMoGbJQb<4-26u z!JIT!`b*|GG7h)RE%cjWc%AT?F z*mf+(Tx2FOp-dvgNtz)xrxc=`ZVExdDn5s6#%*_(x@NeZIO{o&I)WXW>{o52R-?6* z#b7>JJ-VvA;`j35W%Ej_N?w(8Egc2C`Ek{QEot_q+&FrMB1My}?`<687Z^A^=tIz$ zU|WzZFx+pUsnYAF-bcGj`B>)7#Ea=%fvcT!p8cuy8{pg@T+~zwq}qwegJBu2zm_cZ(*zz5A)-vGt<)d`*1y!HSyFHzlh|78H9HeNl9_ zney=;S*ns4{d@M4!mZkyq-x4tIsr3H-t5wqcfzRjp@Ni+^!h zu4c{y_9M0o+gke*dxAaG5#;oCy>=aQ_vT~8$#fFiN1;$}(k6J&p4~ln>M!a(s(+BH zsGH6xb7)zwyo(vl-o1Xg|7oA3ArH1cIQ&ri_*Ak_T5jH-vJ4I24iLf5U5zHdKC3@OPdwU#r+z@>AjFyboE*^k?tGJ`}wlka{hZ z{@5k|Vc9P8d+vlf)%V+o7Yzn9lePKSzI%rmZAY~{*jN*NJ!HEnU9*{q=S!T!ZGYFy zs*W-zTYiArK4Bhj{lZ4uURrzFySv={VS2DUS}l5P^?vEI$Jftyhv}W+fX2r3aGkH3 zkYDhjG&w7Yzg>R$yK^tkemXz#V#BM}2ctjiD;etuQFRTEYOU?jA%6B(mLXnWkN9#% zr{9|@qkj#E(lp{$R96FI-R#_yPisEvGprfv%&Lq>?`OYrrD`)2St*}Z76jI4sbt*| zKd-v0nm6mr^lI6sSMO6j`gUm^_qg5!|4*6@Vxi?yX@35?yk>%S{D(W9Ma_)p@qkiglrTz8Tx8ZH|e(tUne#I&I zY1x*nemU9#brDaZw8sj)QX8UBL2UFT_Dk7r z`BJ4y^OrVFH&^2=x9~%(f0h26b?nva#I{%Goi+aR^U->TMjeVh@$EI^t2duMm`~}4 zw|p2MJNU{l!{msWKaGDIf22*f*u?N??NZy7@*m2UmTV|6<-aTt3a;gi$!Yvi_4fW# zug58mn?HH^WY3#6g-OClZ+E1%P2_+d$MqfGY~;?tJNn*fR~dO%FNiB_Eh+~UrxfQ^ z?QpYfL&X72l|IO`lRAV3USVsW%D{4;@?BL~mUH~q%GLUHh61m2?^cE+S-5>}>8h-g zFUQ=Te@=d)+mYW66z}mpRQE!c7hioERdvU6c8i35ssZ*v3rEVwX$Q&L%A+5LnFD%g z9=W%;S1N|72gqiNuc;KlQN1#I&+F=kAFrD(&c3kw>ZtoKQfYHL!>*_yt&j8*$IP9O zFyvzodB=oi`mjPxF2pqLXOd(=iX&Q2liuHKO7YMsS_y7T=hEohZCS}V*Nb8+m)J|` z9PI+HNk)}-iLQ!mWS?GaO%F>xao6kOgj41dNq-rSUOL?)vFu%0P7})vulWtPw4B>~ zWveG0K6Us#HYc>gZ)s3W(0o&>XM)eOFfRO!zsmEJax=fIyxWIyNme(+QauI6+oLY3I;?0%w&O#eSGkWi-tTbd*BhVC7bhJ0 zciHv3FP6TWpVi3OGjL&}{xLr^P3bVD)94m=BZDFi2U>mPrpCSvgBAtWg!~e^J|I%( zbnTKVG9BZB3nD*uxd+`5-K*Ub6;|>t#nRk5~}&*grjZcjWwrC!2R_ zAJDa1*Co&rwBdX8`GTVAa{j#Rb2(>x`822OpVz>YeWM&7OSI!AYtb!^-6+n9gCvb~b!vEl-58P|}PQ&;I1zzfjBJ<7VPBBZEe zZpA0nr!849TSM-eyto`?cEjwO8H?V$dzSG0;meM%hNbo>jB~DMOO)LVD}uI!Y%z@o z_LYfDBG=z-6;qk7`jpy&rI&65=C6}x2t>@MbK zD%JhK?rptqPBw>FVyy?QlkAgi4wzAFwY9O$Wl1qVsHv`QTJxy-J9B{Tz5Sr8Gq(o# zrH64EZjWoVBiDMwvfH}A`X|iu^SyJ2JDZPzarkm7f$j@DC(oG&Y`DCk@<(N*(yn~2 zoUd%J+#=ruj1X(tY3zI^keR}$*zcH+^fZ`b@L$#@Z>;R78mDn-Q+3<51GFa1J#~_* zzA{K*R-91wQ1(%HDei#IO_Y_fv4CSlKs+DV{me{SPrHS;9Od5NN^$<~Jm(zZ%5)XG zt^oqDIp5j8u)nllvG26Dvie(>S+`pzTUuMsTkqOh+dDXPPOVGtiUhVXn|m0ymb=ak z;bUOdz)65F-vPJ4DrP7tM;T|vh& zrOa3C3}z89axgS}NdPPe-_tf=NBKlOh5r}yGLWNONMSy|AO6>g)9A)bG#x@;q;xQU z;X%HU5G#%pmI(fUSZ%;Q5k##2N^J#ypNVM#b4sXbni>ks z3V#EdZw~w>t7$(&O4=_ZHQ+$%AQX!ndHPV^m9?z(i zg{n5HvCLRL!PU(<*#5!2jgFF^QX~QA-`Bvh*vdKHy+wqXTa;sz$7MT3lVEVz`62R! zFdts7dXTaiy&Ga~@AJvbD^+(juh23-^1fP1u>YFv)ES;^%md)m_=3zDE=3BPEoF05L z>O8ZbJ}mwKtw1F&ROnP%-3yPq`rkCO)q3?3b*g5IXQrpW;fcpA&22?Pg+lo=SfV(c zr$>}duN|i9q&%t~p}VE4^7vIZL^DS9O8yo&QX*gmMV4C3yPTtJ%gmjsyH+Jv_?OQp zpI14r)UR-O-u|q_EU!;l+1<0-Wj;^a`yng!aLV!5_p*I0+01@@j%kTuo_4*r!Zg-6 z$avHL$B3a}+XI3%6X3DMdZ@ExuazOdn)QNx%{O!U z+Bbo|y=0Nu8RtIdQuEO2QxzU%Ul#Swo0WYo>qw?7qaaoLGU9pE<6F0TUXOa$UOqp3 zKe!i60hWO1u&O|V?+35*VPDjp6A;cmt#B2st=!}?TGiI^j!iVnY!fd#d~)1ZtB%k-&McXFwwi4 zX_7JC>%M1^ZnCNs+Z$#VT`P8!+WZcn}cSN8UR?Dll)pF`&d*VI`WwIwvwcc7`*Z@U3GhFE=pdW>?KRxoz;@8`S8V~GrAmjcEUr;D)hqeZb{p5+d za#OQ_-u^!sH|xiH1^P5I`RKowA9ZQ1F-|XcGv_3GXYQDg4DoHNA<{jc-K74_s-nI81{2N1 zFf?eYr&OAFN4N#-LvMrv;FB6@H`;%bisxl>$}X0c75WQXAwk-vVjN(XjqYLPfthb!HGb12Z@;5~ ztcUuZ$9?Y@-$>sQ?`W^P#(#Vgyl<+HaOD*%iWQ}&D-WBq>`z@Qh4nBy?rC8^-Bz8g zuki{tt~Fd$CDOe?$M*zAwPmu!G6%JRpD4t#3lw3>rpgZLUv+=zJv{GuoKfAs`eV(Vnz`l-^Htj~ zFe9#$Z%TIuZSYiF!i{ys+ZtAVQ#Q4$Jr|~VlxNjl~gWM7d76| z^?Z{HrUu$Me&)b6o>|;{Tc&MSResL4)IIN0bMDwWD6IyQrysCYKJZ@W-_^gsIKpcX z%y%$eyyaM8Ia_14T(X~VoV5AcX4;={3mB6!T&pvTH>P-ddk)vEQr?x{m)B#@00x}w zZ0zuJbaiC89tnx`kIVo%n7RaWfmmcIDxJqeukGFwJu}o{%EQXjDvhd<`~%F#oGQNK zTDrG!OXwr=P-T&9AJdSDlb=z30i1g$nBOQ1p8*VuYU^v;d*IrHw#hAZKC|c9H@NO_ zr#K((SLb9~faQjHiglD@h-;sdx36}raon(3Ee|YpEfXyp&HHP1)o82FmR~MASAEw0 z#&rbd4M`9>iO-yUY`N6=pbz!$gfi+zRG{p-y1nW&^*uG5X+SIOSF4v-=U99kd#I19 z1FCqcxhq0ErizEzcblmc$`a*Vc|T^b5CnWu-_bA|6rW<-YvCNR!0`o)DS)X~F(Vay zR5Il>#ckyTb(H45axv4NzsgMz1VEIG%VjUJjF*i$Ul8pgWS*%NrcJ^|0%4t};P#0hLY-ilC{U$wFmGf}lwPka8Y zd918q{P=w~m-VdkC3nN!oWDd5V|C&Ux36$fmZ#QeGnJ9D%fMy!6RVeXmrqgzsFun- z=|+NHdf`EWI)(UVFqsx<`y#q_yGs9gMb06lXwy4o2R=EBGv}|&};O20&E7Eb?K|}B1qrEKGPOs?o`#Ta!}0xtCww><%&7Ys&+JS?y*(c9=d;|k|~gH&&NI+y)LQJl%I6n zynpge(ofYaP#gl}S%;0L8wl-$E>y5saknd06_M=75h}fHGaC& zT7`O{@(}bI#>@W#p3w1(4;>?Hao2HQbjR>4^&Ra^jTh_DyO zP!XnhskpByRhv~d`5IYE*+(c1AK4k%O?Epy1bn|us0YFTp;R0TJ%=r#ORx%ig~`Cb z`3(5CwhCUrhU+`J>rX9fYH&?b) zzCrc~c(-DJ75X;wHyz3>1^%zezzkNOISGtgM*0eU2>7=i1IJn^-4+mJW9X51GY#o? z^jda`tevd4tg);ql*=aQrwPn-U@^;L(*U78fVZutM=%?p$8rp8RXDX65dM$CDnS8^ zMSj zxdLt&ur5yK26Mc7hkK^GyF1?fuRDP|!~Y4s^Jd^FKPSkk9Qq-%l-UKc(3kDW-eKpn z9oa&*LRKsbkyXNc^sRv-I)rsHSC|UAC-BP^04H80@QQM@33`(!U?%&)fT4@&AmCJ+ z0St67w<-NMFsZGF+3j~R|1vf7e(3RBg8pW6W)Slw*wF^`T;P=|27Xc#bATQS%XN6m zF6J;3&%C2$%yhUe2e`&&(&50Hl?(k7zq8xL4dIZX34%mhYk6jRpBhbzgQXVecyUjJwHQ z;LP0D{3$*_xGeOAuMWCU2Z0652)(rTbW>&)vkp?00?RJgUd6!MBb_4SadVGJ-Yk@1Q0bP&oO1A-K#xG&iU=VQ570{*hdHOi?mt?>h83VmT z8570$Kp8|p8J&hR+u_Oyz)%-W%V;_M8pa)VKrgR9jIK0+ahELkT4D!`Q)~kUug1V4 zCJKLojQjzm&{L>}96tosz>A<`5Aiej4$xDLgT8|w9}Mj(9&lL|R|@4o^ICoczm9Lq z2SCYq@Cv>Z+S)R15tK!1j^a9UN!(a?!a{yN|0^%>V}Kuyfj;F+U<&&k`j=(Uo7_(; zpx&WgnStyP_8suqEn?@e{n!Rs>&opLu;4GW~^P2fX5Bwo8^_Ej+;A%5dJdB~_ z0$W@O#Xt@VfsgM?x-ayJ&D3jPm^%Tz!aHy_khVZC^;Z}P3Waf?0Ej;51|vt?VH~DU zZTvwf@IXz0v6+3WgS`kIjWFQAjH6b|$5 z>;guB^v?qhMjN!wa^TDV1-PU4!gv+(4!45wtPH9R+g0&UnJWL0HObq8J?SrtkkyxO zlBw7T;O^9crzH}Y{+cpoXjdJWXY331M|M4Xh&8eu=~%Iu@SD&`xX%M)r+5s;%l?5k z_28Ft^SCniT=!qDkIoclf7e838~b7FWlK+spSezrre=9f>zXFj11lZnPs%@(4J{j6 z{zYYY^G>x_F6^z?6n6^@f9D7Gnz6+=~f)hf*z^$vAg zO`+z1CQJQ`DplE5c~ZVirj;FGqu2|qnSH|60X?&o&0`0$KLh*dPwa6RKl)8>P^Ktj zlzkObR0SD);QYGSO0}asK;K9$GWzfkBXhL zJM=Uz!#cN`shn26sH|UET*Zy*OID?G9M@Y+qo>FYD=q-@Y_8^$uCpP_JIHj+xX{QM zzcaoyRO$-Uhn2ki4|V~a4-)O9me6|WFUP`&Um~>r=M?y3m`$?Vijk`Bnoy4qp2G~w zVNRsqbmi*T@?YqK{6EgGZRgEhYc5oVlz&swt|&dfSI(=h4XzTf)NzD1_a#!&A8-Y1Na zrklnM#*3y`KK~d885Vg&=oYH2GGGqocXEr}PhE}NJ2{OINzGsqWXKtMY9UZYXqD9C+zfHz*-XVs6JZN2S?Jt_inh4EYWeL+rNOYCkN~#-{ zCKa41a2BpE3M)=5=PmWz6DgN$nwrzC_ssB;d&hb8_uA(@+Gm?@vR`t*+>n_OS0gII zR))j`{p^=xc%l;N5nPF*tMye)qncahHueF)BK;5af3={Q%~T7aj=Rvg*b!@wwgy?+ zS^l)Btx@Kf>VGSqm;O@JE_YPcp|qdgQ}4?&dgPQB{9Vr2N3xGJvpl1`pZe|&80CM| zWH$^plzYbN>v>%ExaO(#n(qD4dy-c>z+~GMzVvpO3)@mQu_Ul)SpLMkBZc?NT2zm- z6gbR+8Ae}!Q>*ne2dXDnyZ+zx6E?_`NO}MpAVdRs#JL?{ZPzBHMeQda|QOcr( zD(7#u2RACEg8YW+q16T?GevdLoH z=k>kkBi&Tx0cwD=i^W*gtSqnS@4{{+W#!*i-?Z*@#ZXUWo~kaIOTZZOdAe>6^)B)#NIQ4HeyLyd9&&>A+sU2HM|sm?`V6u9bGa z=CE3;u29RhqE@Hx~xv4;$=-o`}gjb z;&S-}bu;a8?R1Y+eT?T^eYt+4*BR4Lzw`bFgK|R$h7S%42%Z!0l~0amYxN2yhA+0W zmVwnFm4B7%%D*Z9uKbI#MkR%XZ}J!9CT9Jc?s%V=8uRYin`3Y4y}z3=D7#nw>#}ld z5B?UjSefiG!Mlg4m&s{T`R?@@Vch9m zL%{=sn)+9JbGnWSr#RiQq=v3|Ut%ltFW~df77Q)Sg5SG&A9CzjUuJl{Z<=!I^%rlZ zygl;X`7xm&r;@Y35f&*%d(?O>GIlb)HI6b4_o6-Jo=-gwc&ZJVUil`QX_@i9_j9k6 z`UKS~Hcy!1+GL$rT~ax&(yz)~oo1cpdL=xA8O*ln)m|UGJNmTn6MW4+O?>D1`TP6% z(*bl)pWv;*=Ytvsp71^6#c8+5w+aVrBP-Vx|CIkMCl2PS`ZKdv7R+D&acg?F_x6;G zmrI_vdN%ik@Vfr{tDl;e8tpTwovIwq#lGhP8V5xC-}1fUbI`cV>r1Zz-l@iRKAdj@ zzpr6FjwElrK1accT6eB(RP}?h@ugSGT~(dT<89a6m*@_PH!2_PFps}IcX@>wUzy5% zj|JohFA7Tu_lo#7;+u%IVXB~QKBo=g+EXwi{1n@YDoxpr!UegfKaI$&|M5}!&JX_Y zHoo4LJoo9khu=Kt|ET}-tFIn^xLxqfGMU<@ZfklQGB|Q!ohM;SL;neB5ZK#iol)m= z-}kiND?dN~1-@O48qa2`G;yYFZKb*7X~C}Cr`fX#oMnFIO?EwBAY-)s^>LnGdhIoS z&c0eo*qg0JHNi`vSz9GAAyY{o%OAClY(RYd-!Thxt>25 z%%;|X`$M{h-3{*)Qs#F?uaPG?)62ij_s({K2e(Jw$TE#>t0pBmy9d8huQ*ns9Mn)Pd}ssCrhH~w)R3TA|BrsYIMXbD~TwBSho`rO3KtPfpN zZaxcq5Oj0)rCVnjT=?te>BqmmdGTqOHB{BXXJo+O(8M~M>n4Sr^^f)~_uc7#%70nV zl+a$`O~Oxy&Iml>IYutyd)YF|jJfAN^?}*gWyN(%ORCSi;(*V3EqhK`s-wNW_vse6 zKm6DFswQPEN?QzT@u=zDMjxU|1HbYm!ec1cjaouNV5rx3%G4tW8R;-r$AB2AfxCEpp~?jbJIR`|Iio-~-#;zQdl?GxHhZ`Y>9sRqtEmm=PW zto4ca+^5^1`NTA`_pJ;s`6>T=M*TNkUi|Rn*QAqob8pD6&Al3bwdZx+eebswC8e(6 z%0}MHBPTa-MybPm!rnsJ6ooeqUs0!J_(;Ex9(lCUo>sw@RTpZ&iQr&b@Xxl-}Cv`x9%u}g8x+ZcE<(x=Iv zabaV~t-m^sfJ7w64y1|8=@c;zoy$wSDD_l2dtWv;X*5_%1Mc;JwaQ z+Mj=QKKQcY#*T+OQYTexU~cJF>t^U0o8kjD`1UeA4;T|3>=`jE|so~G{>*fXLe>TH9du|CZVP4>hTH!O{P-c;Ra zYGkkAT;o`6A-&2m-{N8ZzIuImVeY1kM{mDOZkLpBGx%z!YmS@o_p2XoPFa*o*}qfV z)^<>bd-(Y`37F>7+qX|>Ze&eRim8+Snc^9p>E7(vX5C@lpF|IF@!NZj@G9X_YgQ5IN^)z~pSA$zYRhzyxEk4H&R7m<%>K7_b48!6uoUqeKuw zDCeBDs|}NG)sFWFf9JdRxj$fbo}KCGP~H7@b#>2lPgRotagQwBEanVsQOvT&m|s{1 zal`mome*B;`IV)@@rI8jXJ{sR9SkwoX%#s(@=(l}#*G`r#&oM6+%&E6v8eljTfO$` z_Rz31jx*TOEg99>B}{gg#LX`!#;t#wRkOSa$9Nm zPTyJ28K?T&{qOmBHQ&~9w5k!jMSfyW7BiUGy@K53Q z!vZ31#@wpU#F(OA$F!+;BmAI$XP?%d-E~XpL}{lmz#d`_Do-jb&B{(He>3rQY+Qq9 zA4wv5{l_xY+`BEQikBZphEBTJ>e(57p`=zXW>+!mNT+(y%^Ci#!eBD1Iuxbz=>zHRBD%=rU zaTgsr)z_W^%qUK}`{@R#@2NVepRysUDy^HooobmhiCe&h3W3sCY6bJHwvXYSXFuaB z;}st8ujsn4rD zG@07D+E#2DtY@Qs_@UG-c7-Zb!)Q}1U1b3Er7hf+}k=H7xBRtzg6#G|fGIy`} z#4@$|yQ&UV(bXT+%m90=S$ti7qvO2&vSS%vSJ)|0^cwak(-NLNgPHowU(6F#SC~P% z>38a~bk(|j`bS_Hw$QV_7YVbi7v4Eu2Jav*cdt5Lp5C6`7mQJ!YQqrySFn;WfSt~a zp-+=$A>H1z=6>bcvYsW)OBR++E?rl=BmbwIxUA@`2|2^_4(46YdsZ;IXj@@vVRmt7 z*_w*->Vz8J+Qv52zKjo%J|Y)TKe0pA2hK)|x)rMfw(Qzq9c{W|(R_5BsBmfiJT=QG1c^4@8j?D@!}#yws)2dtE7=wZYhVU44$ zb)7lBazfdNqOgM7`BU=i7Yxo*=T6S-n-Q4VD$6TpL+q{8_{V~gjP6IEl( z`kGm`mi9&V#rzhq+K@_HRa-SDHT7Z6y;GMAR+Jp>T?|t_c6uB#JaQlFG0Aw|OJk~M z+G$$l6YZPeqxQ)#eeAQxXPVC;Z{9e|bA!iYgO5Sy{+(`t<`2~hW+2#JUm)lmdDcZW z8P%2Ly3$_@3-gZU-z)SiyqPySw=~C){Ym!W+}n9~^70G5C=Mx}QFN>X*1al1tJ7-s z+D_OWIDEw|(vRe8X0_U^UJbiLNZoneE&Vh7cKv0y)$aG*>$@kocQyQN=6M_s;8_Us#Y@5K*|OD5b=$bbG0Ld38Cw zCoo^BnP+`zcjGys3t0K_r#;ykwV)ZPZK})Gak~2Yp3t5f-NFnD4RP)p+#7pjd&GML zdeX*v##zQiUhj;d#*W6L#*M~Rp4A?WJRTYX4V~S0x~+EmP4BLordh9Q!2UoV2CE#I zLOkd1pzTjxxd*x6FUm+_3#>4+1-I2c*a3JG!f?g?hUBJI%M+e!5lqkK8)B1;8p$C%1)e&D_p_ zy`_BylgA!ItRdXvg28O~!lRW(q2Z8Wl3|d$)oq+R?6A`>)~jJ(jz!y0yHtHp^(*Ve z5OfK#N_r~DHd`B}xw^2)MKWy{N^mwCZ?RQdk$;<8C) zHRaZd(#kf~fz|7)pO}Zk%EuGy0n={xip zdN1sl@}w@q`@)f6591;1A1{SBr>FV)d^xw03*uHfs_m(Etz(!y)3(O;7p&^dvMsc= z0XsiMHqkc5zTKX0Pj?LC265kTmE7-O@#P+WObnDX@b-BW?AD>dMvGM%OxTFklm+ZZ z{6J2n>QPszJZc_IQu*XkY8TlPRt>gL11L4T@!dtXqPCJhIrp89fw!fT;LE}}uzFGs zb|MNv)A?txmKhA+O%d>RwFbVL=m~qj0>P$A6s)RDBZH}~uxqG2yr)i~N?{d3Pd%V0 zun6>m`j&b}b%d3mp|Ed#4&8$~32){d@UD6Y#BWFThFw14#98U3s3K0n`}51fYQbGx zDhvc0KF|1lU=zj$YxsR(jPc@lN1}t{+VOE{+O5}uW!2{lrtEJ!JoofSFH5dpMB6`8jqcGSxH63=~ zgT;ARQD_fe8MK3)VjJLF22uJ9-ajt_+Y#}w7pjQB9U1t(Is~k-xPg_AKzOfB!TbGY z@SeUB-Zq~F3m3j%O=c?CvzSVp0!udGpr0llzTxNrUpTw~>kA~<)o{QUY^4%xDuFMG zK7)6I9mp^$A7b-T3#mC+8`%b{8$IEC2)=ok1-q==z~awFsUA_690YcpvWcV8J!r)r zVI^TU?8=%AD=ccdjJ7g2)i<@n=qOS{?WNx`W>}T_l(f)e=xTPCYAftK^nw-e>F|Et zPAw!C!~UW_`8|9m=?ttWJrWm-W5i$Kjr&*B8R9ZnCMyT4i<`wq)JUoeF$Uh<_af8D z1juV4*+n`5s}v((N7!(%N;H}}EdMa@a2g)^xZt zVlc6eA0(b3czzG8%v=`shz$i#c*B2?_YhAD31T<_)?9>XVleD*y2Iy@XN8A+U$B9- zR%k0#@nc|5+ytSU;|$oWX$dPnG5k{aw&P1!PaP;k3x|oF!eikHti+qdo(#bS2qxmH z;7@yS!C<)~Ra{E^1v{};a^5P9Sj}$+8##xdca7)16ntTySu5%p|FaM!))lhBO5Quz z30NR>2WxE0`0Hep6e-|-v0gk_zyV7K!=*Ig5bsW8sp7;c;TORn&Vx0b6GSUvw%Cqj z35$4K@&n27hBELT{1Ku)xdFbEB4NehJbj*;B}FP9?G(W+|UtBKZi>u+uLbY~g%CEa0p{d)TQrglx(^;=HI~?8icT zt{PTE##2AglepzvcU2*|P8vfF6uJ;L)kN`?G@Od#CaGtUeq1NgE=19-SejcdL@;xN z{iGYLL5~DmQs>~^d@3I%bzvi=@A>bgL(Dy@JD?i?d*vR(p1e#U1T4nfB)c-X#7lmd z*o|pJZQ$2PUz5|hRbbY^mx) z?vwPw-{K9nF7+K}myEE35Y`NZbHMM{nGNC@K84!CtEd)CefU~uHLTZs05-n9fL0wX z6sxMq5zrbds6w!~`4rYd=1K(w2X=-Q5X;0sGMEh^b$kR-O#RItl%7*7!QzA`e5+GT zb(2;KH>Ks&HR%eul8NGD#dWZXmkBbJ6IzoLQz4!pjP!73hS*j}0~rgACyIdkE!0KotEh3!wT2i%LqrPMYijSy{f2&1Yuaf7=LU&YL&4wK*T z1B9(ab5#$jseKlF12jrAkH2e=qqR7t_Gg)Hc{r+K-tkTz8y;FOL=q zTcp|KZbB>M@JX;Xu^%{V4B^J`4zqBa{Yc6Xj(|PDEI`V{ja~$Mhl`Ttr$ zDV6pj4%92TZKD?SB!=Ho6myXX>X`G zkn$yQkI_N9wV`jbTS$giNe!71nsSNbPKqzV_T^|IMfigAVn#9ph=?lP!~O zbAHlLx|oR*@7jNnqFEEQgPQ}AaRk)nSKx_$%xY1?O%x|Fd5l?zv!9jDu%}^lK2BIF zj0M|`DMA64OZYRv)PC3-B9L2{{o+LEMV(-@>P5~IcM7+}JFxF0fg1?x)(K!`t0~x3 zWZ)~b!Ng?-M--kT{$6=4MN_Y*nZR23CaiREv>`LAhCkY(! z6l@6p$^FXxOr$c+$#byd=N9ak3nO2F#({ybcjFa|y1S*_FkZig9Vu(2GcYbwQYUF9 z%&T;yC&)CF^pkjlxGPq}zSNt1uwa$Gf&SAOMyVuNgTF1k66O(usqNBX_zH13HHzsE zJMadHyT~MRBUsN`CH+Y@B72J?crUPnmLtI%JMO%AQ-WO$!YeLCXfLgRea_$U5yDZ} z8}JR!@b`(qOf*G+bx4MK$%-ILb9gPRreA}3!bl+mzE5YsmSd8TDmtW4_?k`)&qBP^ zTeR^jxdmbnkuK)*<3ux2311d&lFG=1u!HyK$*L2eaUD0#vRXcTz`W+mfct^F#YrKW-$ya02Ia59Zr1pCIdyesC5(}^im5oLzmnX6!zNkf?DnyCda!|z6A5DO_E<|Q2o zdvt2R!rv|u?Xe$(wpa#h@Mu>q6w3Wt8U|s%!yf%B@HJ{*7%`H-+TBMmi{4K*f_)DT zG6&{+5wH*86zrJ)4Zas01svG}=8kuyQ?OHHCD_vY32gISrS60E$?>qar6G(3Gl5?d zU?e{zO@%q#P~sWvSJ6wqiG8FtVxaH_#?Hm!Bk>6A#^?nz(-mO1?RzPJm;ko^2Ei=x zJfPVac0CA?>smPWNlU?}0$OJwXuQaP9U4o;17KZHBOV6}h?hZs##)%u)+63ZG4Q1- z3v-vNL=DWlI}>SOd60uy<4;i62cSn`DUkrS^q#}hNs?4UtN}irO-53;V2=4I*oz~n zX|Q(%to=c5Phkc73bgxW=rhaV8|fzS^{bxR3*QSomWC5`$T{#0!VGwBki;)w-{}mg zGweq?05tu;Hla=I0VNp&J6NosqvRvlL*fo!4j+V`*BEw1)B|aHOj(SQ(})4 z0nZolP)}d@o_HI~$aaWk*jv$y+#)@Ny)66Tsm3mK2g{ndus69kSW_Db&sXE&``c!c zAnIW@{uFrhk~mZR1#Ch84z|=1#SZYbbPH&A7IwR}f?mFWcn&-E+rXF08$p6J2HTR8 zh)-Z=7R;f*!XF715&eK3>PiU$YkD7&dqED35f8##`6)bE+J#lH&+1Ef!rBGjGPHnJ z{z&=+^puQ;a$E*pdH^_VgIz8WfX)H6N<8BXVh|)ky?S=29!@&08erT78&?-8Z{q=xW18c1C1cVwkkHEg2)i5jG z{g1Xm{C?pJz@-o@j4hPBz*glEkPzEI6X6)JE*%P-guB#wgIw(h&r_(c;W2RbpTKvU zfv1rN?NG}Y&^s_4cDy}>bXKr{84f$|=0JV?U`JeA=!s3?nfD%iwXPzDfb41v-)H{< zG9m$<=B`8fDtJ;tn}}CIx5rv&`2gVL$xxp85N9|@u$`c_VILs$7s#aN(0?vLTXY8M zZ=uvC`0}MEgpY&1_8cs*g5(1Wh#{~S5Vgji0-KGOz*eOXJds_4XHEPn-T~#;fJ8hB zsB{8Lt@Q{7p2QR3E&}ZGT>uo0K>xc4Jn}Pq%l;Zl@S8LdQ2Pp=*w+Edt)M^7f|@P` z9_k1Ptp=PnLOY|jVbohQAM96lfv3MrptBC_y48axP!mxO6bC^~hC|OOguVHlq2I2B z^3Q>`3Ww2UC6r(#?9AE#J2`b>(1@~wo? zcslgS7I1A1_T&1&_xET64Zrtq1ud`^c-byx5iemc+F;0e2h4y^NDU!3JG6lcp0A3) z8td0k%AG*xIJEyD$SoAww+8x;33L;D2bh0GtcCrHE1(7sp-u~-ody8!6~H(5UjUYy zK{{T8Cy}YJcXcXA$j^ZTqKS22p>Z-iC$|ClhkC!lfjbJI4UPaep)J9K@Pr%#)zP2-+bQIOGuIF%V**F3zt&YOa8MS3vAd zP{!+kLkaZYVBlvFddD8XdK(;lV3!{W`Gx{s8$mxpBya(0?!#Lnko4646 zjP8W@6~mxb9{?w$!ToCB=xZ>J_5sZYeV}AtKt5jrC76$b93UZ<3Z!@>XhT3di`xP9 z-=H6X+ys1=L)-NQeu#ncyZ~H0K?e4N_SZ}4VuF|p-*TkDGu(9Glbyg#BLT@srwlv- zJ@PVeegsJOW}vBIGThlAw_u1d7Fw$byhAwyXu1QB&@fINhu(A%YQGWsKLHr_g*F`x zZ%s@f@1{a2hC+)K0CMX=+FS;V&}!x?=qXd7Wz(S^_)bLye1~=&8$<2eLr+`>JPGf| zKvxC}EglMf1At-|*hM)Ep3Bpr6;1*_T?FiILv8VG3~G3(f;KgSE!*}Wtx})`0-$`a zfVu~8pdIA%WvE#a+$X`vdmDNy>eN8XgeK@`s4uu5pxwwhrlRG&-vH|ifF|kGBohH8 zmND!!;4|K&??I8yqlpHHA{VfVm$^ z1hj9f0%WXEk89v7YqBi_E{}k+puUyHa78UJsId+8v7r7Q)L{|?`Jp~O5>lhpR@4)M zmRp;Hd>IRz69lDs4IGjJW$FT`H3v!6!ijDu$i<<+$$rpk7XXcPzz8)XdO)7OfFM%% z6#fn1P7fuc!POh?M?!s2S4#{~8w*@D585{rIG2Uoh6Bw|K*I#}MazQ0kXuu@lQsNV z0Y4f_NkWYaKz=p_?&t?p`a(u{)_3OIy-JQ_nDEue%$A?E<7abF0Pwa3Ij z`BV_2GlYKN#053sAf-XzXM#GjP>yhjMZzD;uZMfopc4Tu)Fz19{ZW6ZC-?+InLGfK zR!*c*a|>z`@&$Zl4bcstOkJQ96G7IEg)*Q%8*GQB5EHGNT7h>g&{omT5+lMmN}}GB zNibLH46$@jBDCp{4KdL&stL;06lk`Gb2zj>5aftDi9Dbzkx*Az7dhHamG4nb(h%wg zbwyh_9icAuoFxy3^s=s+7`UPa9a(b}>N-K2EQo(J_@RxT22jI}fLBilL9|e_Hfjn& zjXCY0F3P?HPl;YNMhs5wZsDT7y8Pa)Dq%P<2V=4ilb zD7Y~NQp0ve4d!FPeqPCF~6O$vMg;9}O{4 zYdGpc$M69_2RT~S!GpRSQ5#VUs2QfmnNClrBlZ{6kcIsSwIZR$cGQ4~I+9Q(qP8Zq z;((ljqb%wYM4dRY-YC@Li7kLuc&*UaP|qc56H~#sh+2u!8XD?#!&wDt@t1Y+v%p=b zw+ijg;*W;EONS_?Bp?qjXTGRO2<<({ns`tjA@9r&t#EpRKLI>uaneN%hAiZXwpws( zmvw5RWmdF(Eb9ox(#a)2%Mtj)GRnG_QG+LjA~&J7ALKiXgVs|~lN#1U)~+V&qr>>J zR!_`H)=7s_Ox90}F_EiL?-1(F!}O@d4Yd(r9G50O)I*yF_p&eQGDhv)c%*}@Lu(|c z+Yc?D%DN3@eb%Vc`T>lrE?tFZVNCf8WEGxIJ7pH?$^OeJGi5vA7s2-|9H(JKI0^Hb zqae3&L^|O-&N_WD>^Q_Y;f!$tVxYC$Jur&=1_%281}P5090Sv$E!_ifJ_zB5;Cuk? zcf%RuqP6Iw&I9l8IO$A(0@At4iC0G44sssn3aB?+)~9&RSp!$SFfQr`ms7|# zJ7vw_vIa(UV;!&l6AQI4qE$~>Ya?oKbZK$Kc&INN^SccwqJ~J>${p%jME$*ZMs2{d ztqj!YhZ+;porg>N9?nWoJ0WV7Le7wNPsv&?Wlhqk@3K})EOLm14qrQDvI~Qtn!=8`5SJv((YloBd)1f|5+5W3+ zZAR8K=Q_)pqcH?6wP4M#hoQ!NlpiPw@GMIN?3=YN`A)u<e^4aP9j~iPSQ`P=lW%{3` zlVj9|)}}yLZ4U3#lJ#!OwUtx4(*JMqU9nwpkgm*ea%>m($@#g^kjw9)C)Y(j*J9wR zBgVooS6N(iYg3^MkN5Gf%|VWj`M>{jN;KHC4V}KTPAID(keBYw3zp zOIt2uZEV-QoU1F||L*(mlF4!2kBO;WIldp)<(6}Gl~cZx>w<0bq z%C1lsb=OsYUSe{=Nvq3^rpbm&uCkAJ(7UhPj#`~Dfj{=4+G<$0ej#(h7n>~{HK zsQg3!+IaGvE3T_mT=z14JlCdhrFPMCU0rG3&&B2Ey1L@JuIOKz$NQA!6#t!0PT@-L zy1HoK{l96VM0}raZ3>s`{cA1ONCod)S2vDy;t{C$EPrm=3zVFBQ zf4l$RDgPb%?>iSNE??KRHb0k7?G;_MceOFmC1d9bmBZe@bESRX&qeM1_WR#b*0u|V z%CY5muJgaCx_tk)t1G4}&-bO8D@;x&-?_NJ<@bIZ*^TFamr$-(ZRq=H{<}>7x6uFJ z^zWDK|3<4e*MG;ryZ7-#_xp8uKQ@NFfA7lIH8RxZp|}(T6a*9m6a*9m6a*9m6a*9m z6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m z6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m z6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m z6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m z6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m z6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m z6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m z6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m6a*9m zTnJDE>2eaHM8I9`z3i&J$B5K_Vq;7U`*%17%OP^u{~r1uBJVlmnEx(AZE0%h$aLj= zL`jbEeyM8fP+JZ;b}fzfvi+B{Ha|Iq9O&|u%l7XYyJ)(u|04I#O>J0hiT>^9N+st+ z{tuKf^uOa{T)FjX!(4TD`QTM{zn`Yyumff-oL*)?n|HN@80v zfxa%9wYj~YLZ((r(?v;+gK1=hT$Ekqlzs8;{X<3x&xnhBAWAMATyD9oWVFzYII5lP zhOLSi%Pl6irHs1V8q`1W(C_`0mebVogNr-ltE(ktUP3NJS1lLHe1~ymUc>xlKRJ$z zr)6&cZyu+dk_=Pe&&AiZsa&+=Fxej|V{dTfDaVo1%NSstNT{VOYfu)r(#zqsWp$y2 zy+H$inMV;(84;x6!c^u}#9QWfyvIZ4V!0JCR6gF1C!;9$P%Htqs;eDb)a0Bn6#w#p zIm-u9#~RBu!y3C<1nVM)xoU~|$Zm{>KS2^D!Ff2~?2tI;VUw(q1rDq8YL@V*l&YLp zJXgV8IUH60ag{rL%A^vh4E*tmG4NUlN3ru1!}1(~Th+&X}(F7$W}cBR2kJW`#1q=Xt zD|Q|wP8VV;yO3@f9F@*{*%#xM{SzBYfh~X`*fQ8AcwpHuSEMSJPL3)2V;Y%ep);MF zpWL41kU|c}T#%CMKsVAW1YgV_{R_e6st?v2QNly68^%SKOh-ss zz&n@xs2%%ota67id8{#j+u%Ih2~W7uMpnmE_cNo1qVm5c0gmH7CHUt4-w`kTTLi z+%bh~Mj+3IkbhiqULKE-|6C(6a%8Q{$FWf!9pq6@9(M>QCp-S>5!eT8Fbd$k1KikK zuy2^bEgv{W;Fw|gXRNS5j7m6gjF5XF$^`6Va$m!g98i;cBK9_mGd+$t0)!w{%u#gy z<7V2jN>%s<{D*jl)>`Lbu&9hj&i8C#mTS8d59A7Dvl1t;F5V<9up8r zjDfs~{4Tc$x^XUsZ6W`$56WDHZH?`yfvY^v!*-VYrQEg{5BofF2l5ee9&)1mbYO%# z4R8|5A6b^5Tb2N_q?TnlN-l#_wjt-Bw3KC=OH!F2)By5PmVbT_M+f&vQ4hJJOhWEO zis*-Vqn{^GL|Kl1%-0vLNL`L6r$h=^MtMwSMev$P*Lv4q3DMg z%Djw{L6!u_*Ro5Vi^^?)Jd83zo`qsN$!+CoGkGMJ$8I?k<&8X|qYK--Ru;*#SEOZw zzdRE_8CT}?!Sn*Sg_O#{M+aCTGKiq7pXvZJGsI9_?%7`o?in!XGWkxPyoo)*9@`SonLQCPS(Hqj4p-cuM1kw-?&l_SIiF%M9 zj_`rtkFpqP1wsm>Yk_>QUg^*(ItWF|c|Zj@nS)wTaJE4m5CNPo6aszZTx>;@_+{`c z$`W-T`~_fK4@!Zf9o7t=X$)|Wd|@Ek5#bP0Ecp_Rq15;s97(i<`ei{4BOw++yq^s5v`bO$PXrRR{RyAuNj z>i-Jr9YS;=-bxPvqeet9#P)O2UqphELamO3`N_(1po_&bz2V0iyL%XNFseOdK(th0G&znRsBWf zj}>_p+RAzrcgyEjo~=4qnOOc+g|5bGe^%3}+G2?jH?sH#-Sy4!Ah z-D=#tJihmQ>oL<1X?Wo=#Ar7>(zntNcFT3!q4UzzQ}0%XYF4UxGU@a;F!YV222*qC z!_+fq-yVRZFZovNA)b-$5--HdToBiW|B4%KA8EZ|d1OvAXP5)4PL+?USY0`??3Yq? zbqnjU%EW?!C164R1mYX@9Q{bmb2eU^?Qz7o*Zqd3 zrP~Fs%U%cEUTOQf7a6X*&(sgr{-FIp-%WR26{nh^O;a1`bF@`$Q=O*jlke#BY;)!* z*^cN-_M}pX*Wx;Hr!baV=6K`qb_}VRR^7e&FY}S=pGvnEbuF7;=~3J|ue@xVHKpwP z^vv9WT#0zFq^@uXHHREHBG>P7k&>{Y3fI;-+zCKDM_S1OloPF4y#grgu;9`oUjF#9?C2ew|8 zIaPhDG*tnm8HK+UCznnt9F{etWOvQ2vZE<^>0jEf3xkSRrrj=GOb#M$Rn4e;Lv+-c z>0?rT?RMi^k3hH09$)z^^Y7|E&vd}6iO=f*L*PcQmTqf3(*q2?H`O=TV_scMqg3Oh zX4+DZKUKxTB5JhGsJ091g%eZ^`;@FBG~mYa%cb_hHd}Ygnd({9*Q$3^j46&T98mOQ z;my2V*=3nWa#vNgEvruE-nFT0M?{sjPU~IhD-I`9EbFW03nw*8RnzE++B#mM$CvI; zy?*x}641%Nz27LG>Au?nF9lBV3G!@i9O~27ce>{|?f3e1p5GXv)g|mL?KTx&A< zcksP$_&~Exd%@7%ZJ7EF6GCmGC$QUCnkp4F(#KK;xt3_hPqX*q7CAJwaQmM%UKK-2 zj+aK2=a$6g_sFC(3JdP%JKkPQcB}4JlbPHvc3bHLda89<#^%gSdxFMFx?6MXA-ZzU z{qC0yE4{S7Qv$sLbNvPdSt1i7YD}$7_rumk&JC~{?iyAF=tK0LotZJZgFbm~^NB;0 zS=U6fRjLt=(RWo%s9bvk&PsNt(mBa|sm5S0u|=6bE_qP6tvDLaK^Z%ef6eNedm*`H zLbrllmNn@faqBZi@I7tqvpQ#-tZYZtAxGHiIeMx#8(O)wGF&v-{Eqp~2zVIWJG3$V-{cb0=S zORC3~EGrBt)D(=!&d%tVzAgPm%GvmQKxSg4IxXkv(X<@<4r{NR7MaG%TS5-8Occe7 zs)1k`2E+%<58DtqFLa&PFP^{o>O%tkvh}_?vwM4Y zFYOLZsAd&CS$f07XcsWu#4C>OL?Jbb=;#_xRrmtRFN! zxI@_S=%PqI;H{5a@RTSv{0Gxc_Y>ZSLMHkDsoSeE8@}^6rYU9OR9~|f2rqIN6T_Ag zbGSQPFtJ3Ug^#Q&%%x`cn!hYxm$c5^nbWo~uHeVCZf}gKyR*%S-Cp&|IbXduBjwr3 z^c34rd#C)!toX7B@{;zvwA%J7tusCGUFMbK^`-y(u%i*ZLW)CfL{&z$30ms6I&6HM z`@zS(R~no8g#|A$#cJy65F&&wMVh@KwXe<(7o5q9$k$~*OkJNkEn`aR^_O*D2IY;ZTA68k(&gRN>eIIU1-49U@wa>c{X*<5 zeyq+nbnvJ#UJRHOGC4FQ^mfSeh^h6zh=~eb>3<`9WL<4|fhpbdk#BYATc2djN`0bN zv0FW|Ewj^okLpMJ8b`Tml0HamRUL2LKqk~+bDL!EPQRV@ zHvMtZquBN@QgXA+Pf{B_I+ifnJj~H3Z&do;f<)fJ=7~c%g6ZXP%R9@Y_hSP82wf4G z7j`6UZhc$5O(CE9O^7fxm{=#l`>Oj3-yb9P1>DiJS4oEHo(I);=yEod=|+qbR?)wz zA}D`rqnfqE3f4^=QR!XY*_z?lU;TI9-%m3J+H7kMH+z zeEJY;a|@sKNlLpyf_SSfb;MT>XZCyNn7VkKH6;X;hTep5!hmS`6n0io=K<+*tGY zYDZ0H%bn6i`2%xj1eRU$ z{wO#gQd94Th}eMAuqKgL!fO1@ro+Aq0;B!Ebvv&f?eVQgZ%q<&nvG<>CNEKc!3&FN zgk200&x`k@$x>%-q|IT;wY9OGtXx@eEpJo_Rn#-9L0Vi&+?$Nox3cmp3>m{>U%pvc zYOgHH-H<^RG_L8+U$=$WKB7)&duSuvTABt2uz?TtV?cUlbK_piBA)+kY5vC{6O0{TMK@; zBe?Qv@yXH=f>rin;ulAL z%~)cSuE5aWeT&BqpUc6QgMaZ|9+(ncQFmF;1XE$~i>RgHoqV%A#`sx+PxzMVZ)v|b zTrsrQjZxoYZ_%%5AI6Cq|N9@Uo(XdWriPw@m$d@9coB zflYn#eC*-sXs-~xZ=B!Nkn5p^fhSD+jbC_Q^7Zq6;Lhm=Yffk{=*FvuGr3eCokk0E zC_92qkbV-nOQlQ|{nTNx#yWoF|F%x98de!x{=9T^areT=Tvc{JMrrz`jA$SjV zYH)t=>ws5AgU6TNvwdcGRk{z+t=1N4j%vd+n^gWRPmQ9JsQvUM)gIa?Oy>8IH<$xb zvLn!T)ZS27VmDVUF88f?Q@*8qcJZ9O=;)ksa2uCZ>vcD8n;{=C~P_bhEg?GJ8U43FF{YUZ-f)I0Rcbw4v- zu*+2!ReE+2dsmf4KUAft82To&p8Z4h8|%e*F*~KMbRfNvEP&_aa-NcQ3WD&&?r`jL z_zGw3iPc|Qdsrve^tD!6_ElS|mYJJdpK{a8K~>wU18Tmp&$73-q?$+A7difR^mcUR zW4H*hF%c(ifjU=FztLCed#ZEn=gjx;jQOLgh;71FXlAJb*avibcAz>&b%z$1F!dnS z8MXmCg!R#!)Qne+)`Y6otAA8W>nHSpL)LgKJTEp#Hb5Gl*nq{^wj#D+GZG9XAxm~sc z+#X&d_UG@355-t9nkXW_C#R50$W-bHsbls~7Z{O>V>U2<(4Oi(%z0+7Y8~5BwUc?t zPGt|W=U5;1CR<+>Mdzv3F+OZhRXs|r+Q$0PxvF#YkMwZb22WrM>BY1^`B>^o^dXGW zMWUnlEngtq<^SgGTs=pT!^jobTR60iM)tgYvB^NLTn~}47lDC!-)y-?!}+lO8iZpBm=3R=vZPQ-GSn$Nah)RiTZ)wPd8#l zP-%c~6kSGFQy;QD>5rMq%zFA;dI3G2u~5+r$uyw;pl=fAsae#I~Y$ z(j%^!WZ}-iI}?GgBhC|o#JQrE6e9RZzT#v_5{{79gh+T>lP311{*+czH;G&#kUl^r zk!zTlL@U;hQZaRyaa1uAN`*0>Fzx6cr~}mB)N;m7&7(#W1@vF!P+~5=0at66r z@)B1{f5Kas2~wHRTHMN8#AU({Qh-n(92Uk9iH7@M?*c?$1>;^^&SI$KW+m8Pl=3a!ch@YaeU5{OuF z7Z*W&qi3f#6WRkc@SPXCfg2-fkmZT=X5&Vc> zq-3Ix;109{g`x1SEl}DlJr$;r=f!Ts9MU3wMPvyq8Db{6oTP|2vQ&h3 zsMK|0my|%=lWq~O;2l#8wL}oM*;%?oZ?(rz?U;PWr}P-%cgjQhKynbF^Z|HH9Zm$( z$A!PdmqI%tS{-WXu1<0Uvlr|`RUO2QM25pi`0GVWlITZwr-t%%9CvknxdbkQu?mG0 zLuNqBO;$~`CX?5x>(Wf%v^DCT_HJYcvRd4)zGz+sJRB(95SNpl^i`gs=MfK>kEo@5 z9N%8mO&mik7Do^hrKwUMdXo4HyUCVDlFT?Lj`wiSF!=^&S(*iJXrUAjG`u@8{aDHA`9o+5Ulmk^U2VPqtoLpD)2 zBY5j>(oT`|Z&bOhuJo5wWcQ)IA#QVox|Mj>X4Cx13F=^SwV2Kwlk8MO=CxFp4-+J+ z2f2ZKL7fr^A{O4@ek@i>O-T*ag80k&N@7%H)C%#oZ9R35ZA-@RlZ0`kiSDktY+J=G zB`aeFck9733{=U))X%7s{x?Y+GqJyybqz99118&oZA z+!K)!4Qyi=tM41!+zi^E+}*r?@!D+Y>3z?)xpy0nsowRx>lm-XJHxT=9{L{c4cugFJuWiF@vu*7ycdI|BN~(BX*0A)8yqc^J za(buCN~%hYC}>`pUa%!^O5wb+3EUUbEJ_ExVFD~_-qud>T;Mm`@1ReXyV|hO_l(ax z(-{xKFxKO$cB;mxPE^m-&CxEW=TP@`CHjk+1?r2MxB6|`FZ3JTHyU~xBR$u^Tj`hH z!@N}bPgNf>edwKhUpr%IRyME1FP}^MEZ+38(aRZ++umD#@AUH(sb>pn^5&)Q%4udX zvX6D!jcWqB2Tly81CspLMV)Gx6cuGU?f$vvXwM7oxw<9HMBBG!x9WFg%_|;N|7cxb zv$ZB!2-V#4eCfTxcedY0!EB_p&ewGoM(&B}Qa3tipU)QmV*i02pR3P-EPiV1S6*GX zCA)KKT=KFvb)F5pH~03@TMhrt{xk4$(6hdICDw(OIh7PK$;&HbaNPl|+jn}{EF-E# z=&q1n5dn2rzk0-js@Bzo<|7q+(YpMH*)!7nCjDC%Ij%45$xW!M&`Yq-1F&uwOQ z2xvdEMP;Md250J8BAW(I(qB-0uBuCKbM&>OT8~;=mNzcwn%f|EW$w6iYoa~z^;^G} zbMCgj{qvJ^kEcCN$Q`S@=T%K;rQbZ428;^6P~W%BA5EEHn|3@omCDgw(r&Wxh4}@4 zmb@%#RFGQyS%t2uuTxnWMN>Wk98WQF`v%?BOXJzO!3WW7iSmlKfi1GIqb*kNjjb8JGO|eU@BpK?kLkq zXL7S@jWla-RrzIkP{l9Bs;ozeYhH}HYxqlZddlgB=hXK)RH2$qXISPIYzQ{&a8K|I^8Q)3n#vTG z+RP=%&{o|N9zWfFfB4n-6CuB6{L%Wn<76^R`rlyZ4Fe+<0=6PZ~Y0=Mi?o_qwOA#}%Dk9ml&@ z)vM5RQ<)je67@UpXQ54lTll~73l6#rZ>^qs1i6>0Y4#XZ&hK-)6zs$v$4};^qtIq`TI4Y*1WXRkPR4do=7>C*IeCEi&&dZd5E3AFqh43bbT%+jZIg zYs1Ot7xgaHdlKOmFwyIwn_l&o_#6KhNV+6yigZF8PxsJ#q%lifE!#`D>_+kL9&W$2 z>&l%ofoG*lD{jT6?h89Nq5G#ouR3iSw!IbI&MV7(yZdQN%)5H`_)C!&+TLl9*4=Vc zW%GI*=o8#H;)57NgOhc)g;)DDV%}A1^ZVy=IV}sWRd=NA1~=~^J|X_b@a>JFn)Qpk z>bXXJOvTY@{95~BZi8fFC+U*ice;weC4H+abM(*6n^k~+u)m~ z0gbNI>lFQIsJ~x|$7Qy&WoVW*@#5Pl8Je;x@{-9n>|~wc(QE2Xi&jTJ3+wN*jveA? zT=`4c&?+7OA+yRI-lK)4hc5RI_xaSgM!!sWTG2CiWn!ZzkM74ly80yL!HK&=Zfv~N z^X|EtM;)I{&!5qKiY*9}H>;bW^%u>IZA1fO@?GDAz}lgk6MTc&h6)n-D2W|4mc|Ljf>qiy%iW6F1x z9j)FioYO@FPLDbsy(jvQI&=Lab&}AhdP~9Q=}X_uc=h4a`cEds2E8hIG2`);8^>=i z%3e@s)5y#j*CzxGj_kk&`d0iAKlR0xL^`4Eg8_eEyS?nqC#4ChW)Vs4N_%wdalN%V zY>?`KrB_8}QD*j-^yKtYnXB@n%(Qw`(6q)QJ2dVzsr8-cd!FA3rm8I!yDE)U{+3_( zi_9AR@7{yLhSZO1$ka;={7jeW$Sir0k(0bB^}}>+>Vu?d?|ytm#7=vD{OKPzmfmb$ zy1&icDTgOr8rHC%y{+DTQugI%6XV{+r@T1vz;?ga>y^1f?UDLbQ3pC!^f}rixfy9X z$d#A&&WlQ0o0yoAnti0^d)ZQPq`qnQ` z8d6r1UjJ3)^C5|^a&sz9*uvOLrkyd*+HB~2p}nn1Se;vjzSi3rS&5$MdrCZPtLUS~ zh2h;A{L%11Om3Y8VZMH;Ok%~r?9<6-66d5m$z*dj=WNX<@={atUv+;hUG==^`|_!A z`hd+-9OKW7nA2@lKy>NHFSJiDzWO-j>%=c#j(_=gMs;O>`j%h!mZn}qK0MR>NB>}= zsHkqr>(`Ux<5S-jE;2vo$EvOwOY3ZCS<<@G7ZE$p4&GIjH_Yq#RBl22dT zHn;w8^oL`o3_bs0`+7_GCkdOMzDS@mh9_-^+y7!!s&DygsfFoIgO#2B?7Fd zXyDl}F)FITXARm!CHVPhFWN7bJkRKzc>OV8)Cp2ZJ+djI~kdwmiK z{qzq8jQDog+x}d~LZ9T)+<4EI-QILeUY&H|?SrH%xqGVZ)H?scji1pLDk27^81(wC4PE%uQ7WJ-$uKlvMe>T}17NL4uvL-bysjrMO+O56;5M-4NY4E9W%<^MY}cT>WywUro#^n{|`}D0p3LR zeaGWT(^wT)+}+*XWpUZX-C1C9cU_#tU5YFP7I$~I0&S_cY2(T0|77`o-=F!S4Kr`v zdvo8t=bn2;Z*NAe_CCoOnKdP+Pr(+C$Dbjks76G*jdGT%TzyN`qs1s|5vH+scJ}tH zkh7;}5Z{X$X*wT$tZ11Mg-cv1G}d~My%|(HsUN;izkWRSB{3=SljhU>FP@~*NogM< zU)6l_U+bJ+;Ky!V`8Z>)hd)*?pUg-PM#Cq`Ty}=4Ym+Z zBT_f(yE?WPcKuSd@VY3kaWI_~$jvF7@*t_<*A5?Fyx9MI*85SPV&4Apy7t@pZ>qkp zSe!TGGQx~gQ?)as6$;#KNC6&5k z>!lgW-k|i-&fw61#oN3f$um?I*?8lm*t@0cl%4oXNn07sQ>D4kR$4`@BxcH62v*a> z^u#jJULx*p>`h0$LC;8mA9*t~PNjDFdiU+}*K^*ze#bvM^l;hZZcmCm(m!qpvt7 zL=4dsXD3i3pXfZ6?oR!l^zrq(C+yP&PmND*Jlgc=;iJ*-XDZp%S9j^#+FheOW!39*AKG=hzh?oLfFMYl;!L8(N z^xaef%q&%uZmoX3?wKx0PZ~xVJ6Y#PGq!fB5n_TeS69;9*VH$nt2Nhjjv6BrlADv6 zp{x0uGS{S}yj%RZ-u*8R$2=%s*bNRjb!657h2eeO3A4@g0l{vEbT-eg5rL*XukS_;F9}9)31g(@;JlBXURVh+-d0+%3_nXejENcBkAgSeM^GTp{jC zuf>(5NLlGtY-_D0@{VI*^aRUR<2HRu!wusn<6Glu<9_22^%=q_Z70f;3-~7ai?jOv z`0I1U7p)%;gFkTf&Y?R8?#+By@cpWBQEhvZg;jIQwkWs0bm2l>9L22~Qw^C-%X;z0 z!$wb@ya^`n$=&ar89F5A(4uioY(g=icyv)AKE+`)&(JN_Zql>bQnXQ;BEr@R>!s0@ zS2bEU#W2Pw8B-$6mWQ^D_MWy6k@HMVv?G|hRx<#^tfM=D%>jDw&a~+uVbz=4|2t%CLhmz9`wzG-8`&B%YOg)BvkpEO%kkEq{+t#c=2*34;^@%5wpGVP`6>ERpAufDt8 z>uK5d!*lx>iWhrc=!S8GJ-%Fz8Wk&Ml%83faWrAZcpa{j?j`P;*;`YXv;~O3$UO(l@0G8BZ;z z3bDo22XuRkHsfBcz_sRL!Fw?Y3Epi5)pCAGjr+3x1N&y?6W#rqcRxHB^`i3EErE@Z z3mwJTt5n6fWtC=@yImr=Oy^>8rc^Ne?sIW$AHHpFPTIiqgp91rueq^oK1)d$D zqvS5FFY*lthh1X(#GNj5wCJ;9D+)h|9&Rme$WYO$Zt6%jOQ|hQlV-~dv77CtcC)XD zu}UBE3tgI7E3S5>I9t1)IYX(9zS_Rk`z*d(@~Fd;ckgZ_moLbYdovb(d#Fsr-$lO{ zSsS~k$mGJ+ZEK9Yu~|ev<1v-Q-*%VvUJLZ`4RLpJ@5}F*L*^y;+KXG{R)mvTuk{+Y zM4ortjH?)bIW9lm9Dl~%EYfRfWNBeo#XVx@u{D_HbP7!~!^pA3FUkn{sx(@t>-)ob zEWf_1e{RZ;nkiCp+a&7U))$3d=X_SDP04cy$k2&`iQXjLtJqam#(XL2*SPD}RhD8g zmgv9D3pJhDKY$I@p)LYRY2&-$&dJxh5BRqG(|rkk7hi@bq`s-IV<>4%HlK<*AG;u~ zWy}o6O6$f*zg2Hz%!jo$O-W5l^)uBy?hdWUb)*RC3D5Y}-(7a<;>T_tb@UofsR>I^E~Pn4o*LheP*r@-Pdv{VxePq%t+g3;}5Mx=P(@B z{b2Sgtrd%WKO_e7LKz`4P|y3q_dHZqZbCMrUW2PxPd(E##Ihzb*EHHx#$4I1jhbqE z6WPeT%=lEFV7hObrR~FXqZWe~RGxq8o$30X_e5hFP9hK8NBjaL_bfYGeNd~`i~6P!EiEofO=~Y}No!W*0rT%BuVJX3 z)s$ft6Z@sJ!OGs21zU3yGGfxIr=Cr!@!`Pxj$dYd9{cI+_o`V*8S%+?vOfy1h&_RA zffn2=<2S=;9b=BP?XlLg#o9|p)pKmJTdXG|o|^tNoi#4e)M8X*Td;O^i?jHD#ffr5 zzFW{N`IL2XE2$M(mJP8-xG2p>Lt)b#qt__HW`8mri+Ey4Qa4fW&>ZAS$sdui{uF5C ziOIEQEz6AkF#xM?Y;yd!58t+B49Qe|dzxIwJ(xe}?(6&(%%iulcgT_KKgQGMp~g7V zOvi|rLiRr_Gi=a^@ zM%+M6N7WzfE!7%rH~mFq0n2JvXf~=vb|2l1Oq48qZeY1ziTxI~B&AwL8(#+SK zS54;1sT!)EGx5rGsk59VO!Zsc*9zL@Pt4wy{viEv+MASxX-{$oXBPc7C3UB(oi8JA zOx`z^7bL{_pUSWU=W*SAlKY7s?8N`|859@fpK<*FoxQb%aN zGk*g=T|_k}#tY3v9Wg~YC@d5gkvr*BCyBCK^a-ux z2Vf*umS>QcnFI89vLsVZ^*39duEJDd?jqN-OL{I2;;Z=|xt|rZcOJ}EvX11G%_#Gu zT~-C>pBZuAUuBl`>?&xOy*n=>5G^$4uL{jbhiaVqs>Wz27kR@{5ZT1K+)+8k66Loh zT5Prf_6rtugrKdfyJYB~AIUx;b}^CcWa6rrAhX0^X|GsMu0&WBLQYeDLq2mDv7IbKB%J&PmCvp3x&`m$O#Jhtx^AN4@O|US?Ze z?Lx9pMR1F4s1Mv^ja6$io{c0#+=jfhH&?i-z9n{BCX8|ElxOE(tMpES%emeG;i zE@mUx1ZBt_$}B}M7n3d_n|Dz;Fa9a*BHoe_#6Gacqrf5G0`F}zza!L)XTdoe4kq&b zyz;ppayH}?&-^Wea<(lXGLEIpbIuHW&flCp*xf|Omq&sFbf3GYNitrEv_)<) zpSIp~IHS%xB-=3SG}|_Nd24P&1(V(MHEbQUk8s@-LoTvBbAl=?w-ARYo2dgtg!oFV zMV>{@qp&P1cjfcqC+UFnJHIjTz@Ou5?xozToWpY)=e)`NBj-WJkM!bsqg`(@l2dY> zWXR!?GXM1ylKTqQU{(1tJ6)B|4O90s_Ohf#?v1EuSsXP!+Uz)P{bq@=@3U{VEQlCj z$^@zxt2gORsV=aE*l7AO*^$0PJ(ezuI;tz3B@Y(A%ZcOxqLea0V&!@AY0=9!=5vEL zy)m97kK!tow852tNO690z+iW>u8)oYnImL9yG|2c_*I&0tOPM3EM88p! zkvlFSzeK(}PA(+R1k=8oJXks*&6J;tgZaB5d&m%I=pFB=<0_N?EVo$Bg`BjU_^i`u z|7PEFm(Mf(*qr&z-zhlTxy|`H*iMO+4@tMFk*Xe=4ccFggCdt(Mp&9#YdUn%r=qUg zrdvv|ndWn*E{5xde8UgzD{eMBgl!HkgM(h7ycXumyQw&`f!IcDqx2_cDh7GE z_)=Ibc9eL&Tp-t5*|*1A2i%L;CBjtwVXM&^Nsh8pEO*8x%+18$9 z|6u)L8y9^mHpd=oDPc1^RJQa8k8z&iq(N=)sLOH^SDBkm9VU$QQIeOgOMeo7Ap>4t z%7(R$k`GId#InMEz9C;bwAEkSyVE_o;IDj>Gco&d#=-2-3;AD-4?Ue`j73dRk1}xXGf2TI&NKQ>*RQB+aDP+ zCYstroHsPqOj0|vY1~GtCKX_|QAd%3Eg<~lXL+s|lKv%jDuP&5bc(s+b|D!wwz|HR z-lCpo1r?l2a}u*+bBgB(Y2{Ol*=60ca%QFEr9E;_@ovhGb@p^`2-TG*z+U`GtWo`{ zuVUss_0S zbtS{d#YDcGAfA;bqZ;O*Tt_^`ujAi|oy3^n8qYy@fv16&D`=TbW_HQ_oM%gKmh8?v z=jxohI+;u#@4DlS$RCnh*p(QZFQ1Z}{0OCx`mDaYZkBGOX+3Q4UK?RA8*@0elB2J6 zaMXwx#s0%Q)wI~$+PW_yOWj1(R<~YzA8WLeIz-hcC@?oUPAhy5_TX!zI zVzw%EaN2&o-F(_s(z?mg zJo;L^DLT&Tv(s^{V>jAF(=+oCyKJ4V|3^K`Fj`+vb(>b0=8T^hP9zWpvWyZT&6i@7 zPRbzZjxe2X&NmUR@w@yno>A^sZku~Vo-4Ck*4x~2S*GM}86yj9dA}r&POah^=*e_e z$e!XH>mMxIrBOkZ=%slrrx(?)bUEf%j*hn3)`I8{u|dZW>({7i@hhS;%`c5tY`3C} z<~OQ?nn=@h-9_dzIhoNig~%j{Qf81>l?0)h@Rz(+*()6k)enB>HPW=u4R>qzVDEKL zIA<5i8kF}T*O_wVdyF&Qb1|d+S3WDpv)ffNcW`cudqc3HR7ZHs-z8*jgXVxXUq3v; zX}x3HY5Cx=7U~wa&-&HYJ+?&LKaP2k(;~`QR@rYxsNv7m)_qk^WLq#vq(*t93<6W- zE@BtGFh(j&@Y1K?NWY!GC6y9%-fIO<-BI3Ft_!)Fvy0`XoU16^nL0b$}0uJq>#!S`CqY;I1gNh2f_kh_rMs*A@%jm zDH!M4hN_{}PH)z|tV!9L^t~xVvkjhsS$}<9pMJ;t)Ek-8AbVs%?*Jw46VrsEaz5?W z@VdVGM~14A_3aVXLpEz{$M}G4qUG1mVPm)XxVIM?{yp8CGvlCOPE|D1PDwbvY~DbA5t zA%*y$nyQaCoYLi*s#+V_8{17W=i{2$i(4N@J&lh7PjaleTx50IzmY=>M>H?hb2Tfu zuG|-PJ#kWMN<5{$5g(+b(l_BR;gq6N9`Y_`fnuUYs3;^Vb-C&~Mz>g(Zd_m;WsSG2iM|sz z&!Lzr(4#_ zl!+NO->ia;DctwB`Sk(^@@Ho=xplqCyjN@_ObeYP_Ni_9IXYIm+jP)+)OOuEEoyzN z*|7?%(2|&H(SKN~o0dhawXQTj*A7NBa8vDJ+aa2MDc3};VQExr)sV&soUl>cBkh$M zh75isa9pSp`puQ8T;WUbdnB#gJ{0u*<-ZX6671ticaC*lDOllbm-7&_bZ2VA z+=kw(ISF64r|z3lDn9~}{KlVc{@ zH`*q~)Q;;JrI;5RD_B?Aw?$~wCslQIy)_hj0ygXt@tIhqd=Qi2QFnlCe=0;s0|a|$ zN?>`A;IsYP3VJ!`yBfG`d5^OWrCy7 zW)QrZ>BMxhfm~X+DjMVzp*;V)a9NBL_WIX*|MK4t&i6W<8M)o^CgpX?`;=AuN1yZw z?(&{7>1|V|IZc84?k0H~^Gx3P(mt$Hxk7bnuC}{jo8bUzz`TxYjtHy6aWTGW>>BH6 z+wSPh*xUA&recPdku~kd4G&m5B7)B9D{MpJzOY7GLairy2>xKQ=#`oamqHWxvcj3* zzupJlz5bxDAm5vFJNIHR8X23XD;f^X8Gd&Z;WR5sp_) zW>(5B@v5?0xg`~rzKTi0tWdY$Azme&5#oKEXQ8irK<$(AN9CT$KA(HTIVWx0b72!A^IF;2}Az#Bf}>%zV}@HK|Sa4Q(R7IBGkN+TMXHKtz|b zJ1|pjJJ#5Cm`j^Z0du~l-=t~4Y3VPti*8B2mx5w{@HyLwI|V=gm1jbe_z(QOP_1AQ ze;@x|-{0ODo*D(s^EWzUoRe|~@j|mYQ zh-KtT;#c_Xaq4`kDKwlr_Z|BeUT0JM-?OjC+1{KVA6?V0B z@AoEp&7L_0v}=lQStuCTEHhsk+8`wPB{Aq4B%1iD?$t z0BuYX@SqmPSi^pO2VHx$z{aWGtFAFG!2%sc?IWuKZ+R)LlqUhZD20fbkS2(KNvEV8 z5-F7y4u*oE=KOZPkidrS2Yv*02a5v#i9v0IFHk*LJhU(LZ|IkxHE0YS<~70w;keiu zEB9DQOFSebq8{m^D$yvnpdT}P*vf1NFv(rqInK)6Vy)aA?mbJhex@3G4^jIIpam-W z1sKp(=_d3vI)UCn{U8g#m^(;Urf-ACyN*hyc2I4BO~+D2sW#x)?jY{L(|)gfk{im) z`fd$HE%Dn(K2Z6RxuOUZ0<9Q7T{$7ht2iU${6rt|3ww4E8t^kix?hnXuteO59{ znY+vb=67ZolSkj8pVE)$?Q~iCCTfsBk(%ipQq2$S(#SEPL8TeQ4w&QMGZh%xd#~BE9H&y2|V7(#g$&D z!*5yh&QA%82cX;JFdnRt2vuso{9fqm>WH z%V3*#LZ!2obP_L#>1eBiNX4jaP}(Ww;f=7$XK;fT%B$ryIC?p-3TZh4xI$4pSCH%C zVF9`@NS=*tr94$WBwt6(yk3!Iol-=pri@T#C~K8V%0p-@yhAIS6YYoz#3W(~lmIpn zdk{MaL_S_?mQ z=-#8Hm%+iFhRAFje9MkXGq8%QK{FsqDG9}ZW_Vvq=>m+sGF}@i1JGv!@wp3ry&NMw zOW6d?frH8k^xzjL9E9~7Kq^Halzn;*$9Qfa_7jJQV`%#p zeBK0=gXP3xVg|N_cnl_b65TQ1+oFw?q4E&_-xdLlgP+P>VYQF2hGJL;@e>pl?kjij z`~t5J@%m7Cf_LvR10P{~uY82Y#Q*g6!rC}L6-{_(W`q?w%+Ql?KwF|1wh~~YSAfbx zH5|7gIN42zR{ym=-2M*Ox}e{?{MWLMcx{bu+u^x2-Z#hZt75B+M_HmIv@XK?bfDc~ zRSrAeg>~H`{ySD2{u(}`B2bSgjAMqkGT6fZt0se}4^YvxMLO{OwQ0v+)1tLxd0%5(Ouo7Q*q#gf_QutkXyrTb`xnbSE!uS+cAPQ?&g_VGQ zj$BwVCai`SRt5?yRfRvZI8IoHE38@b^Qhro41XJL%TM*409vEM=dgav|CE2i--LBy z!fIAwouU8rIfMDF{ZEl5d?e<-w#MQ3;rSIKB}_3I8U1rs4lXp{+6a ztpj@vxBch&81a2!Y-Y^Yu$qw(zYZ%dg^wB@)v&@2k6GvZZysghEW#~C;UkU^gepr| zDK4yVQyR}d)hfeRVfe}jYjFM4w{xPG!>Tq$=%%0!5Tl)nIUCkL%f(*b;>stWKl5EF z0fm*YPR(%Q2(^w(QsT&1vXmT9E>iud;^Zdc0xD@{q2A$1luN)vADG{irm<2tVNM)_y<(EpMkcBUcRe3I|5DrM!#Dzjx z<+Z$+yrA3{Dx%hBIN>G6DP8HE(EXsv`pRKg>s0xGvRJyU+>|Se>!mSLXJLeJSUQdM z{DNqPqDm>zA(j)4K!arge?Yt;Ee1cjwDeJ|D9sa_L4V|+SV_`I>*ZOJ8K_)}RA1R5 z7s!*LxHA>i9lMl6du6oUq+*I}!U7y}h&c>Q~n*2;wCPpczfEJ&YQpAVi5rN|;g~kQBz;h^i?16t- z+dJJ;-BaGfx~sU}AZ}02&vLeNHqUFATRZzqwvfYRm(Tg+YAkFZTEhRyX8+K1)nefV z3&dvGXm9Mux0Ew622rOo^wK^QRQii84YHqxuiMKWHPQYjEFc8QO~q3%!@T$*WKpIGo{&d@a99*Zu&JGmA3RpY-(O=>D-B4KYxq<*YM5iuokljmiv}=mbd0F<}^zk zOKQXv-BLD+>?ocH{t_GzY!DnA`oSB8alu~$n}ePCIO#31p1rCqFa}L0%{?P%<2QY8 z{bSu^Z52(t<~FyFK0quKN1*oMrT<$nl3yjX5FOHejI$tCkXI5l=|t9}8mez)YG@vA z-V<5fvM#b%#Am~DLw{W%O%G1boTJK-&6FQ!3a-nyE;A#cF*@ zo+kYv6y~c3clmdE3wv6-E4b?b7t^_Rx^5OUCxR{f@VJZiu_Jap>7dfm8PglYOmCi=I}DKNGc2U z#kTTuSmqqvnxd&bbai$eTb9dEx7HstG}m2Lf7g`I-_?!MjL;0#`E}K_e${2x z%XDKXTBJyZXI8K()pJ!DX#2dN571R8JFym?-aYB5&_2{DILBYax5pE7&+x4AO!3zA z{q8gSQoRFwI-kXR+r8F(#QohhwLn!+wP04hF@H%uSy0M(JNKb;yr*UGp6I7`XsVe; zL~M#E6_I3IpzE*JsFF})+8ao5cT{-%NnDeUA)a_D^b_h!tzoUYkf$gMV`1Mgtx*$c zWP78gGY=g8hU{^A6Sb6ls@%ra?iGhib)Xb8LQYXS5c5!LrlA&5H&CHcl)22jVg|5V z*uUAAsKyS`N9Zkxc6QKtOb;#yHKo=nPO}LLFN>hf6RQ!_J=7<;BHSIeB`Y!On8wU4 z`VhS0NHSm9E}xg~N%s*kToVR~lf?%@b0HFzXt?l}pTH;bpZGn}jUE#$PS+A6zhFX!bdsq(oX+l)Pl@v6m`m`c#8n2R_p zhfbvSkhh@}mW@10s@O>UOUU4t@wtKY{K1EqOVW#8gqmYv1eA zw8b=iaNe_!efH4@nBK^1)}|j*2dVBr3G~oCTSjoCpS(|ZV!vR{ykHqFfnCG&W(G1p z=pu9^bpTOKX(9nDkp`*-%TQgq8hF|TtjNQNk3=W365@;i*%B6SAeBiKr)$&csBc|_ zX!skIL$#(yF#FlvTtii|YN)!X`WV;7F2Y|5jSSTeEevJ_J_K$B)&$xI-UpTh zcL$#b#sq2vh`Zyhpqzyp4QTf5pHbfvs-mpnoU{7+@Z;mM*~-;>xR* zs$Oy9SqhcfIf#}m%yh)ipg`k%^2mLp3DuwbU`LiBwil_b^mRwJWBbj{qJ}T`8(JkrX$p7A>hEmljKbc4lCV9k6 z@0CXI1lpiJk&qT3I{%Jp&f~&%WTXBR>cW}6hT8QBq3^*@!A7B`q1(Z+!GxeW8M+%X>3fns5gj>`(ife=ΜwH>(0Yt)S| zq!-XR)LTTP-7v<}5H+nO&Om7nwYu>0J_4id4S#+Byug4mgpi4z%Z1gg^FoOed z30xM|*Jl{7^IRJ0)&F99v8CCS%nXd$Acm(`(#doVkcGX#IW}Na`azqZ0ok8up}YWw za!u|m-<5{Jv)L)u5Iq7ZRui=%C5pl&A(vmo|IVKeoeK60ya|jBo(QfDdIAjt-2%2? zY~YLkrN6L$fWJy0JKzXb4=e~A4D7^=8;ah_4Ibl1NmYpxWD%+bT^!d>b#@mskG;&j z<|c4uxe?&v{mM)MM)L(TybzF$F}SKKPzsq!^`c)=`>DRP3i!+ndO3RO7@bPZp^j2Z zsJ7%^h|xocf`=jopQxOLqF+gHgw81&(NAUI0h_S7lF%s zBw9c(aXccCO-coz7pt(&zmq@9vw+4PLDY1KXbUgk5PH2SYDOEg=h)%gL+%OYaY@+f zYg`&Wt>t=gX5^73VTFCdv|t|5Q|LnUQtBnyp8N*P;t4REk8)S}gS0?u1f;lu7=d|n zM(8bM!x}Z_&q1^BS#TlxBMG)=6MW2o?^l0wtYjCx=e_apH*>v{y!*U;eNe#g$-a?+ z@4*iIVezwE1yKd3B$6$dzrYN-MW4bLE`%;$IdDs}*yd~U4&bC;dQz2d&2FVeAgz2+Wa-rVN8a4(?;F`d1_2sqMuI!HxQUC2D3 zV-y&!S*U_901IY})Jl2c&k)Z9ctmDjtk2_h_+I!P1%#26D z3c*`}wt)}+RsNp-cfQKLdEO>ogZI6sglC?6fcvexwda;+p%=Mie<<)f|3a7~&;lvB zlvt`YtZE(V0==FsuPUeRtsa8tVu)(4rjGWR+R5$Zx~W#GZgAJw#;l8J#oVJm(*|Y< zvejp)Qgj{q8dl+aYCUYBntp_pc`H0R2Rx8YSV4}F+sU0k2YDce7YGJx{5M3zg~(dK z4m*(^vMm%eR>OX8g;gqs>c*VZ!pGhPi_?p?0Qa8@h0H#58Q7l+^aNaCUol(ifRXk9ICnp} zlEjGp1Sj&lS;+GS{4M;K5bquI?(jT!zi_96?|#$u$+g4%!L!`E(f8KhCveH1<1fsQ zl|s@0aftW~iWNJs#zoLe=#|VA_6b)@ovo5tClkfJQbnn!z;k;+*I@QChnVrqb^0hg zfKPyryEEtMU(lwiu%|h|{3kH|@mSAnVcIe@lR-a6RB{|@Mq_CMWs5ztyc;;ATv#gf4NSnpc;9edJT@7XjBv`cx zzC>?1?^e%Z&mhF;t=+TSUp-%Zgnxx^vG0ffLa-#iPhf#J=z+A$%2?_bdL3oMEG)&= z!1{fX3&Ahm&3@)it2C=ybeMqoAb1?Q4TrNcWqLxm%c7t}n;4xf7lGlqeh0P~#L z&(wul%{h1-kJ)s#B{vPaI}b0;1_rf5St(nj zH^{5;{AB*`(B$BtKzDx&U!HfV_lIYVyO{fq%j4SOD(3F)S?68hwLo`up}$e^5hCQ5 zd^<5kOq6aYN2yKpKA<&mOa-n8eAx*q3AR$=HmF9ZQ&d{6B>dT4Sotr}X|O%jk##Od z?VT^(SHDzSHE&fLxFy^+ zt`3*VG+;_VZRZcnn!?OOx)J71Z>BGMidC2%Y*(xhshkeyenC|O=O3%Csve^{!!74l zBlcOvmSKhg_vlX+C3-2PWm-xSUh(rnOM|%q(LdDp!L!t3@Mzty3QiXk2REgDzNw%m zdVIL+M1j|xKm$9jZZyY<>1t88!$&7uqy60(}4*wxvU8@Bnn=1Pfq5Rs`kT1)=+g&eNkD|b5vuw zboMZ(<~+=8%(pgV4tyy^Ixl|b4~OOiUk0KA=|06%)!p3XaP2Laj2hjg&I`^{PNHDA z>yT?^enEc7GceFC_}$mZH!JkFyab$}^YR+9FMWs|q28?Sr#`9nXcP3u_09A{^-=m0 zx)?1Dd}^Wg6ZE03u|dwsIk1*p1}@c)se^py0Ics7;W3&4+os|?MD`i`iS@H)E{T!o z+jLL51XYn73H)%clpZO_GEZd*ids&8Wp;CS)uT20pxoM4e@$0NU)9jUSlbYxb!h(7 z68gouNoot$TT)d+y+++lRi8b@w1c(n1zoMO@Y`;&N7<+BVt7n`<_0a$8ySviL+^pg zO?_nK6e#~R6TgJM1set1As<`P2OU9AGw7xTfc_NZZ_j`2{E=T!P|}^?Zt5!H?&KX5 zc#kTN-GOqbpl^*iKZIBX`}~catJW=G|=zaPd`ZI=ehTXdD+OAqK#B@8f z?^SQP&WP%3t5XpVRAa9*mDxCU67!0FM>l3V0i#QVK6M3}g^r0vsg5WtPySa*6F%`; z{vZAg-vHi84m{R3-s)b1=aOfTHwswQICsGF-Q$D5*2%lU-!C{K_&7-Nk)ln0CFMa^ zClP2#3E0~d)JXWjoSUezX?AIf>T2k|Xbjq*rUxE_)LwWh&+xd!orRZDlWoh=%7X< zklzlk;ChAok)!fr&g`#TW4s!UbdOxxf`{z|CNd^}K@ot2{z(jGSx~zHbhF z?jEQ_sSSU3qx@LjE$_o<)mOSA+E|60)H!4t-$6lZ2C|qm`3$^*GUP4zAjillBu(k4 zD_CdE;0m-v<~RoaTo&rr79lncVfAeRkNOcZ%TM94EJcRaMf^j0h|=KqT}57Z2Yl2< zP{MzKj9phme!Y?m|*eez}5h^)pVy#FpU zP;*}hrT0$o{z)ZK_Q+PH0W#4Q5YH4--pB``3f~vk#aQH2_9>^Z_e7-c$H-@H1ZuP8E688f@WCUe?0_tYDxGO+77u69h zu7!`Kbu}?Vd6SpG#n58X<;ggfrX;{^M@!Pr#nT=ldS{G%q*`jSziZ#M!1H z&X}zXCXS$ePmz_3hT@o-xPx&uqn#8q@^!=#966lJ4B)N?$I+@Vnp^^VRpB@idL?|v zpuK3-ee_CaVx3YS7)khk1}fyIQ?QSs$RG{CqXOp4E9^H%nMa(${RxU98@)qWNZiJr z*Asu>m^+Z+o2HBhy7p8Vg|>%tc@@yoEF4cl&(z0X2IC6i(V}qvFITCI^K6AoT47}D zUf`_WD=mnFI9fjPtJT2ESgLfxo+sh^JLvCusA)0d|Ch({8=_Cr&Pb7rk;3<9!*kp%~E<*z0?FJJecjN?YQ)ybP_Vr92?I z19yA^M85~JU#E#ray(j9OLHOVcEW`PYc}9lVV2eKm$F!(xL@drG^8_E%al?PHQrM<+jl*pTyBE&J##jF*kPzEwl_&{G(R*3|& ziWn}Km=3aEu1L(v(=7`$cF^$AlWHT~a^(0M}VclDw!m=pWLk`-aX^8fuhy8an6G zuyTGlu!E8md7zAh$?faf9|%AgDZNI*MlTAQho2 z$PZ}h5Z8ju)O-0fYBHr3+et;ZDZ(OlDnAVhcz@9bVzaP8lM-A)B$4BxUwb1okgTj( z>gz*~k@CpF^e)j()`!iwE+1D`02^K*T@^-XECCgLgcu|3qvDC#@^>YftVWEKf^=t` zeWbWcT&L+C_=`y6Z2UvwFX<0W5&l=nNjD)*$S;(I>VLh1NE`Ey!~;jDB8Om?zHwQ; zL`I=3{5SSsP^WnpYD~=~FNw<-qkoltFsiy%aaD1p^pZVPKWP-PoNMa2q1r@Tkrxt+ zBcG>U;^p-5VvEpQRTcg+y8yE=T`tZY z3bc}c)z3k7V>Wr48|6E}tmE9i_UugMDzQ*lsyXJrOP*HG4^#!xdqrBy)fDaskJvV1 zTg2>_LRs82$tHeNHI(8)dvtQpOSGmB1_}^e-1QUGc4h!n+8?vcNn>z1G6^f8mayGF zOKT=3hu$-Nd>8e1=qXN(zCZWGT~*Z&s+{Acf$}+3cV8uL0>4u3rL9d~FW8Fs>q4-p zs;6hV{%`V%KVDTQ_)wM1N3(;N&F(y8!#)6~-UkkUq})w)Pn6)P!IL zvx3sF@4OX-K8B}4GwPK5M69BSWWF$6)rr8>Tn#w+Yy{}AX0z80d`4F6>1zM*dj_g4}gO0S?an4>XxwKACJ z!WvZr@~zZK_=7#ez4H9c9zY#bapj1xSw4lk*$Bu|&k{eA8p)`XRE2~*c`BJN4`Pf` zZ1972zZ@wxA?^y@bUVe8p#kh-aXWhu7Q8t$7#EY>_}0LH+7h%7sSH(*4X$Hiq*_qV zUM@^!&xjM*Z9;z;N*h?sHY-orAEAq+Oji|85+A8U@)@>{)Z9N_$I>N)A3(SoiA9+n z)LGw7bs74IuQ#=uYZWrdC%Ls)aW(}_$hQ9i%i3Hx!OB7r)cm!OTEV-#O{D~vG6hnU zOmZ)!p2`%Vu&NB93o@!$c`(5$@ysCUXrQO|2~i-;lUu`|=)ydaM1D7WLU>6{fy(D> zq6BqNzD6FT2m2QjN4UPIWYLq)C`BkvHzbaWos>-stDKc;5FMq_%6imp_YBokNTx1T zU9cg4Q$##Ul_qJqr#OtMN4AvvW7TGqDpYCaVQ>kxoj4*kXJuKG>JjthXi%9Z5GMn> z$=mdJd9(P8JWg#QZb_;99wwcUpaH9wM&WjF=R&ISaz=5#dOL}&z`FG=|edGABPOUiS)9ajAIp@Gy;@(P_tkiKfTe@P9x z57A7VER82CbD`i^Vh7z!9uLoU5b=O-%_WiR`90Jwr3=_WQ^|kDIi!Y}rBo#k$~=9z*3?J2faIy1}2p~5WOw`iHXKt&@HH(z+KY-XoR&^yB8IFtdN6Dy=$ z;%s;WHHFdQN_nhOhE>bX&?U zt4NyJNxGF-RQ0t}dJ!ZYA`X)+h&IY*avQ10iIl8(q>*Grs=TrjK1D@kH`N<_?lw-N^kk_EF{x{i|*hcN7_dxSB9ysJsC=9d5nB7Dbc>~`F};tf?= z{tOR%F8o|4kt$EbUOG`ziO=E{c!X+rUy8IwZbWR5%0WHg9`4$)RcVGgoR;D+oCjFs zd@EG58Ht(VZ^8~qD;`9qsCUpQJ%zu?i-p8-^6$u;%n>y5E<`SE$(E=Y877|~XVI^e z>heZKN0}%qwMc0xFCt5lyQme)L?Azh$nNljw-YBQhmtJ+2^-NC>Htj<4d#*kWZdNf zczR!Al3axNolJp`d<5V9OUA(~ze*QhbrF5QH4L0yT@Z1(s50yjs)P#6I)x#C+0JAF@dpJt=M7;e<@e^CA z=5z#j?7QfqnD6UJHSX~>81=Ob`3E_K*e*{(?c{#BFIc~25pz~iyx@ynl;6mErMpsn z;yj+$O79VWR*=7<@;*)ZAU&3@%MP)&JcC%nKN43fHN^5#YiMxam2Tla1TFZGv_w|H zU(X626GQx8nz1#x7|bd7t?XqcgXpK&>F3B9E(CvgKJ_<7WHUKS zxlWvfm)HRQD?^N+@|CUP6)0fOCD$qj>AZ9rUh%(RU)NHep_k^#$?{^NF-al1o(bfs zADOB+5Fwr;1!{lfo?OIY zat-*}t;q7^YSMtecgA1ak{2+J_lUwk7P=q?xv5M>T-pT@dw8yGfd62Z)8!D@Di!7F z(orcvPFHrI=D8@o)ho#u0Ru3XZOVM)zLTWMl3!j0Ou4+2B2`3f^bL4~vt^kWA%EpB z3X9~MWV+mh-!B|M?tO>YS9}XQxluU@74f+=N!Lf!VH@f*svUT8DzyUnqY88jaBi01 zP9Gbn`LG1<&_@ZlE^)sbVh!0-*&-c9wb5CntNc`6f!zK=xjdr%&ESvCMHJr)nYT=3 zCC-P%-SOTLy-6Q>{wlJQ(d0ZJ3Om4}-;SE8WW1h-rg}YKJco$Rsy68}ZB@U||tJR<4kfNNB(#<1rmG{~}o$n9CuI#wE=C zEb=_63r1oTCxUBG3A)=0F+Vy3YaM}gvj{O6Xh}cB-9wP?d<5LbjlRAv50#GsFB&LM z#4`=FXpB5qt}gTNoVtR|Ul-LXqk$SFfkAmlK98!iPtry(PiF#evV%brg}=N)MZzcW zM0?AebXKa2)odyBHm1r0P@8oFXj^IU2@K>tVi0U)SFmXe=R<~GKfGU!4sf}V|YC(KY+b@1qRAL@zTzq!`V{#ol;0yTGcrqFF>L>Aie1;C7SK%vo9@7&ul|Gb>e#VIObMV(cSpSWH zZ!~=bmLB_opL#-z@)tBuv#<|oI=FR?d<&)f9^ZsNL6e**jX?Sv?5^jO3$ z;F@ML2o zn}+=&0pQy2!AJej`Yo};`4_1LbeplzVRn^nfs?Dl8slG5Rakwj3ogzDb-#s%_aZc* zhY;s9C|m%_o`Gg@HtuiG4){iaJp@Tgf<4K-z`GhL!$zh&K*Bc_G+lMEJEa|ZY9rX9 z+)gc`H=xDa(KPDIh8?2`a8}AMc|~>UF`B`4w@xX@!;nE z;L1#BEIG6$1>ee`>Q2arZ$YnQST1Qx55?)2w)7-=AkLSCgUVk)=DtIkIoORf723*P z$oDujcc-CSsU~ZIRZsw`EyM~+th5W7;jclF9Q5Zpcnbi3u960WrY+I$;&39#1WTGt z@U!6Jt2fffXoUXct-Gt{LFXke4708Zg)G$JNCB{kdkc3e(eV}U#uoCf&SeCdh8s$m*K=$GTQhy#=6$% z2^3`N*ElP59PM0Hz5=au1JEWwK91flOVdGN8~Dxz`rJQ3=bPZCA8@kWEDZ)VPhu|N zj{5=7+A7V&_xm^C;xBLpY!I}Ebh&x?uD9{N4r) z`#|daff97b)eCa*J$h$XaMwGmS?qz1e?2&L9;9X^Ncc67lyT7acPW=pgTcj(5o;vk zPsdI~73f=r6@*rhXFbbfC*UkxZ=68$#t8NgXnh47{i9TXSyFq%TMmjHf<}-+Kf8+g zww5ZLw_qlk86VYOBm*YWV9|r&)bI8fL~C4EqE{qG^q|Q>IiD*WBfjXm9NS4 zaGXw?jWv=h*oU+dUr~!dp;qXxjWD-L#pv@IBjRD~4m=Opb`^A)1lleDl})gXOjZxb z>5o!KodvVG zXShVjx_q3Zsf95x3-!*yELw-B6r}M2bUO@5w;2+zCuB<_NV1lYW8J}B4?zJ7PJ?Yi zISTN7ABP@)8?BuHDH;Q-(8OZ1Kki3RyAaIN>!ClK!+8BWdeR{95uxHNaMKCg|A6Zz z_&$VwZV_2Q!@ARA@YWXa%qr0I55&s_&kVtDBYY9GLLYn%TFrt_258?2&!JeI*@M`} zv1Za4{NDzvBt4-&xCOfG11}#&T@OR{>;jF*NypKUxnohEj&MCoEkcOhXmK(t*^Tp( z`yeN7LrzaX_+-5GLR#d!WOvla1=JvP4u!<-4C>EEzw3<|+d6PbZJc48f|-m8J8bV^ z?aKnbm2NRTDy|B;YOR}pGC zxNQ$&EdwRbqQ=9J&N*<$TEys!RHs2w?1!Y0Am=Y&y-J0Zr&P=hCg96*7)F|Y^>|CZXN;Y^cI|GLOyrp!gh)V%tl-^c zSclpJ&N~blo(<{+VP;qxwIlYvhobg9(VE>*%TUmO%rqnM)DyjJ0%#cx+I;ljW2?R#1rYgMaj}Y3kiMZ2&f*5dhSMDwqhpt8)WAoP{9IfOu$eWU|Zbxd*k@E{sI|a3Ts zBPX)Eq3;btI|QKwdW2SC9(Nlpe+hLc0L{2LC0LsCjtC=7<{_~ z^*IYVp2i!Q&*HokIJyvdo<(kNK%WG}IF9x|jB7WpBY67@Jz)<@>46cv0DNPW^{8hR za8D$7ZWMCYgBo>F4sZ0TD&Ws=QJ&i1g~8~5-ssch3@tgYnqQtr3F?~)Dpmncl2gq@ za{D8u2ICkxlS=w@3HYS}_^BDvmM{~hk%|U2j{p?|aSuXBa@LjyeG1Cwv3i8mgP#fC z7{Q+zpxP_cC>Kx9(Vj#qod@M_f(Np}(|1AdB-A|(oI%EiRD@4Ot`tS6m4dGjJtPI` z8QDlB9rfl=#w>&pG0rw9pZ6F*fu^`>p;dZ=+d@$G2t4=1a|}vZ7c^*tJ30030~%SO z*&wI5$=6FRQY5F=$*J@-Jd@MRc2KVt>PF6Ole4$vgf)>$c7)3U6=~3dLuyqJhOFeb zMk_bNotzOTC)Df0U4_u(j53A%2v^?2O#eA@$UquokAMZK1R^AnCWL><`EXJ*4*8MO z*?QE$rF@;9#K{76dH9i&$Yd`A2S0MI*p857o!h&dLmCg-S0FDGa51Cg5xo1mlCW3;!C*62Q?r#2%)$KfvSOs;SE0rKPx#A4kWc9`Ag+Bb5Pb3 zE$oJ}XmIx_rzN3@f{Z|r70Bsz54@8+6ch=9#1IH~f~F7&8iwPU;57InK9P7dcwOmH zgn|n1kQB%`>WMOwdJ`WWR3zvJgp-KGAl#&sicpQ5aVHcfQcLxp+B3L&B9*}Zl!oxU z0hAzg6wBLy&{IiWiA_$|EBP9b7U_u!RsuQmPobQor;-?ihn14S8E@M1mLM1gq_v4O zCha5Qjfb0ZXV6mQWIeeEHx!n)JQ;D{qK6P!K=w)H!?zT19M})WayP;;gfir0Kanux zouo`gZ;}q_TP6QVF$>hmMq0#;_&V?$NGS3~#?cSuVF<1RL3SYHBjID>S6H4u!R#P; z5GobHg_McpLy#>9zm`C{lBZNWD{)DD;zRPz!bOlEh~!N|`e_J9q&&%il!(lYl#5V; zTqJ$cUmZ9h4ipG`c^ws;2okoiJYP~0rK}3Z2tn2$`6~4x^DUC|i}LF^p5MSF1y_1` z$q1%_5|YRzLM`%7QY75vxD-w!bWqAiP(75g6E{K`a*@7DF2YYrdn-MgNN>`k6ko!n zL@tqjOXQ`(4@&@ZzRM_gCGAGWqKxv(K@;*uQp-XLBp<>XiT-WwqU20m6Z8d4|aB9fV4oG2H0BO?cge?>|v(u+`pJUK4Hl{C21 z@k^wg@~5~GAIG2a2DSysXop~v~jxsq)=eRpcBQ+UDg=FCwE;RvZvL&@J!PG=epUuP;x$((giV*WdIXLzNApQcFs{#`%iUGe!p zQPG*EGd1GoD2w9i^m9HrVmfnmxO{qd(u;(1mgv*cD&ZY|B!^FPRotC?~tkh&a@n{{`XzU)lm*-t%%F#G$-+uR2`um{{K5= z#g)uW{~b;#pCdgI;=fZMefV>_JHjZrE3qBF|91QDuu4cLRg@TxUq}4UOX6^KruOgn zjuI*%6k0k$e|mSubGUw9qIjV-_Jvm(1e4WZA-qtl-4EbDBML{ z9I1Wo;%o^e1xMbW=HhsErtohU#Yf4-5y}zf^Si^B+#T)VaC3$vE>2fx-AD+<-B~wh zEQg;n)Te&`?dr_Mk>*0BOS$ugmk9jjQ45l zC}AAGN|^thuHvWsI`aAS?cb&N{Qc9A&S&!Ed?PUw7t$h*mU7B8C8Y98e94oerb;}L zg7WuiSVt(uN0Bllo)U)Kolhi;^IeJI{8hp^OGKXj7qvdkouuY?|2!XOx{fDDNN3p` zb2vwc|F7?k68%3*m&E!!=I3w!JGK9Fn183|45?7W@$RILBZtpj9O*b($Pv#G+QA9` z9qaQ}RN^VAIn!{uI@5DJIZ||n{x^jjA^+E1iRCPVk{WS!rssV6G=)#&eCqyb=+Dzp zLOK1NPyeU*|Bmr#xKB$++?}a8pFR)&|Mtz9p7Qi*%^iM(ZsghV{%Ky$@SnO?JXcVl zf&vv3sGvXv1u7^|L4gViR8XLT0u>afpg;u$DkxAvfeH##P@sYW6%?qTKm`RVC{RIx z3JO$Epn?Jw6sVv;1qCW7P(gtT3RF;_f&vv3sGvXv1u7^|L4gViR8XLT0u>afpg;u$ zDkxAvfeH%z|BnLy>vV{-l%JpEal8?KCEllJOdP(ByE7HXNh61gBd#MAM|$M^n1ZaK zgmje6dH&5AL&-(SU-|t!b!VFY?XLKdSjxFNf@$J#`}9N{adm_sq5u2T3~^Qdh@bP> z8J3*Ba=0sZ=dZ(^q~J)KgjQlGzDoYiIG^T2a#X_prx3V(nx@lLdHP>|#Lbzy^6YdY z@f^Mm*H80w=B|Wt=HrOvh)?d06db8K!h9N6N#XNfg*MJm&L>Cej+B&K$lIr3KBt)@ zcSjydXysQyK~Q2lVma!lq^Q*1(Z0@H6uwc?R>F{HN69~JMI~QHOeKVpqa&x!ORuEk z2=9#TD39Xusk<{BhpRI-d2+rv<2uUWbaBM}w~M1z|LyLG;pk7~-I14a|NNKaNib|k zI6^;185B3?yAp~ZdJv=(f{Eh5hEZ@o99SU)SIG!Jf)k=3fRK<1GLwRjXnX%QEK z%R(-K-$Zajh&zd?Ac-h1`#hLrpj^dnLZ;zsayNE!rjg#0TAFpm0=b|4(3)Qcbm z5#$X*6K4w%e}ZkN;5v~zc~ZiVXM)o~(7?#Q7Dyfh$BE#15Cji`=|Nl+91|_>4nz}z zc|`n(FTob^C`UCRcaL&35rQH@kd1tBCkQBRaQDO;@h6xgBs{@&A_!c{o8oFH4@FR; zh&#b0A$NjWLP8Vl5|XBZqN2PJ6cq=Cick5ok}q*12q@n8CwNgL9KnGixKPSH^goy> z1T{rLjtYUFf>foTo)AnG2lh%hd?Lz25aw+It1P{jtcs2y% zjwq^DZS`zh(4#NCRE$OwXGMdS#<7$Opb;B*mOHY4r|juF8w zA~;tB=SV>*D#W{jc0_QLNGO6prJyl65RM226T!k#TorsWC9DH+iy$!Nm*=A3D3NrW z2v)?8geUP8#3l#!lj25VD;P^81i?!pd5}xNVshlKAYv(aP$W%)0Y%&h4kAH6Bv&@x z$iL!G>XZdnC0FH6!VtWubYMRblqhngmOqgvg z98Z!Ud6FycKX>AO7f7BDfX7JO@8bU+ur}}EiTuXmB5@wzorECnkIO?7^iM}BN}41T zL1KMceko;8N=T3_Nlv6x#FvC2PmXevH^)1%F8KnrCe%=BPLS3dR3o$_bS3si6U!+} z=t|Hq$v>g~Tl^|$YNQoO8xhko(Tt3Q#MOXN#Dp|W=r;sei?|VNSE(G;)`5FR#u=s0dzOztZsoF^ z^l2CLOwujF8fph$)zB~Qd4h@J%bki_c( zOvn1=a*W`j5{XC3bk=QozK12prW)h+hfloAps|3#^z;05W^C^3i}Rm&W0+1<)Po6MR+|&^j7% zo1&CNGNb~X)FcO?#9H8tCjoQN2t3t9SmW%8QhNYDa0?J)YXR3ilKL5FzNcWf%nYRJ zrtsYYH1XcBy08n_uy!EoK0qne^a@boOW?5oDEFkB)49MDC-yW6m-d2X4xr})d$uk( z&7HY}c!z;kz7>`)d&Am6H`wwlkZ39f@!C+`q*JmFY|xZSKZzY+wTGqS8-Pu}NghaxKxlp`dBRf4m#|M00oyz)fr;%+kC5Ggcl|f>0(M@^QhnGDij(rB zU#aSh3CP~>qz6<1eOmSwXHrwxx3HgaOG=?SF|%a7&P_2n*yg(4jhPLQSS;z-s3f1~OLi)9+IRGP~Ihgm!-S5b|T z9@!gF>r|aog810>JI|@dX$#piwk<+cF@f!_i=gfHQTCq9ORgPzKwQPcPLKpdW7ws=}qd((i>(WZxIf%&qXc!k~s|3aD^g>PmsDNz+CGA5DNGTAbiMf+@)Kqr zy;e9V&tgiY(efX#O1Y5>;osRx<$2n5bY-EF<)So7-CZBRhn7`AyK(xjEk_s+wx2Os zJ(fSHv1|4gXTXxDUfqX%Xyk2G^>M0e_U80u<9XE~d6McJU0^z-Zfw4zdBJQo9(9eh z`BQu8Uh+*^w!eiHnQ`1qyGR`tcZo+eb>wN{IsOz|rb@6qf|aT(;t)C1T37wpV~=s7 z@h8U2JatPet4Gh&>@)u$++(J39q8YweA95=2lQE^@ig7y8%kQ{iHnv`kOj*s^+u|m zrHSq%qu3!u~MwMkMK--PhAzaQGZJx*$(WN(lL80 z?oZK7z2y1|lF&k|rW&QbXLGghV;kxY3zG#mHjz$Lt+sErjdhI|4~mCv6I4GiX;Qi^ zh_VV-=zZ*ayN~TJxsIk#Ty8hulcd0M#Q=M_{8@gZID`JerH9~Wc|~>Nr+Ea+-RuV0 zY|hg+)e2TOp}MNFGzE4@=kY5*oJdWAd8O^FcOhKM+v=PM@>?5d| zE+hFzn)Qr0ZblV1g6jeuTt}dhN^1D#K*IaMvCuXzim3_A@kDjkTz`Yjquyyt&V^E!fO_)5| zYPG7n%RYjDw3hS38D2JW1+eP2-hPkmuinUZrnS}|C^Kj0E{eS%Ck?_3F;P6i%%r-& zLT(TKOQr)?pMGa+DWowkm^PGAY(Qzm4w_*3qs>IGVzx3rOT$bnsoDBsiWMy;iK1Lz ziJVzu?$2%3Y*vl5e`CBUHB--(A}N92%x+?$gekUBLb84=(@Ds*Oi*XT;t(ZHw&%g# zQ+uYGyo%`{)fXh|Qdm$fqMOMZ>^-EWTrb$DTR}~i#?$w;J0(x+2k{Q~2lYhyi;aTy z32x#pcxptJ5=_|;LPUEFIL(j3D;Kp$0 z#qGjaF-m)iQQ79vH({x961{!$%BiQX?cey?+-fyOB zN+#yL@Yr_EZnq7zUl%v=p0-HwroFcHJF|yHZy#jO5*nCynQxnPCcF8&vMXh|#(CDk zwxeYkB~wf0@jg_THQ6o-Jl`9ZE634o)K@r8-9mp?Q&+uJ)ydV<5* z*Kb@8yLI(gZ_w(-X}4|qS_i3}E{96(1vuQuEJ#2*}^^;X`KK4{LKzeQ1@}sL0DYRVuYLG1a$SqMm7nyZq?d z)xDQtioxCB>($nm^;_T{;*;o^;o<7h#xTUSimSWJKiZ4jb?Tych~I3}@=HW-G1E52 zc(Sa>(vu%A!V)_>j@z$#tDB{>xvX;U;hz|KB#aA_{4WO$4jL2`8`2;s!N0(3r3<5f zuIIH4)Lq!Gs8BgVY$FV`>dlKRJ8hEfgsGvaiD{{6i*Xq3CN!&Tcfzv?dOe6PfD}+ zkG6lzKNR`os4@f6MkLQjsQh%zllUiD54~f>o8RC2;&H~KoljzuS`_S;m%IF`TgYVT zCI|J2{wn6rn45Le>iXAok8B?PB%BRT@L%FyTX&egW~}t#RfbQdI`hK&C2tqLKA31s zwxu1-o>|n;^0Va5RneSx-Rd$tx1yzH$G935Oecuz=ncv(b1rP`shgWvlv z^d8~9)Wuz0C{MFrHXbj!l{YQ(b?U;@j_+E%`RQfVr!AhGkLwv97q5+f9%p>&{^Iqs z7SDsD-FZasvIU14_*}2Bcx)BW<*2)XN@6|*%oy`&Fhz4N+Pn!;Y=$737 zWybd*p<7dbubL0D|1n9480+6Eo>?&i9X z>n>KeR4Yv^UR+Y$7-hUr^7i9*h4YL)khq?zb#7Llmx1~aYe-SZ=HLz?Uxs>z9SQw8 zFwNW3ZL=nki^Ht)BE3Y;7S`HJjX6cJghrt*v%vddbEDcuhJ~K+ z?QYnu87p_NKd~n8>HKli(ZZ0AtBbmpd|#Gr-GIK^-R&3mp`I6gPX#Rr2@0JaHZJ^V z@Os~3_ow_t~s~0YZb1IWU6$m z(lxSkNSJ#GEXf=AGxqcRx6)|JRURyS!M_%U%P*M(^+1;hce_VF_anOgtX5cMdR!D* zxcTFPym6U+DQgmXJ!>BS_CeRXn%FPzE_v|y;h86c-f`@*nuEW%-M(4dM(tu+y@?L; zKCgSPKVdkbRm=N|$EHty9{XVVgSv5f&xa*v6yD)>`yBDH8JZe~`dWfJh2M_yt^Irb zfZ7KukMa73Ia#zbm4CA?&6>v-JuiM+a>abv&e{iC=Lq*TL%a=PzeW9A?Oo)ikk&q~ zuAm;u`M`a;8Z-m|<5T>EKz=|<>J=``B9`k9(Q zYMfCMLzUKSO82x!x)R`;P@KXwl6 zsp;@_bQj%D`$p>p<97vRS?4nKskL8f<32p`d$}WV=iADKU#L3=H?8_OdO(b4qyF{V zRlOPL<@clS0QViLK_X)tTe34NET#SHuaZut4A0(WnXcRHlkT4v*gx!9_@1CyUM)SE zdS!SOxz97SHjHtpqG~Dk65aXQwg%QY;uNl{HbT=^Eot3cvUUB{L256TQjeWpTRdla zY;}*&_of${fBZNg$0dDUsybyzQtAuU(^hf6KK|$7vwKB%tK83dQu3zMGAWF2->9#q zKQ(aefap%ym4B4h<+jYc_WsMa3*LApuTNQ*@^{j&iFaOIe?27KU|wqYEi$9tpynR! zzG=^=y|_B!5%aX7UkZu~Iu}SE=4HHizd1c8>v-W^A=~giiL8ECG!jOHT^5aFtfQzn%dfLxVF+#%Q<5!%SZ7WuCYGN<3NB{&?&!(9vgKwt{PL7 zYpu#w-Ge3VJn@VrvhdIJnB;y5wO<6r-+El|XnI^p+|5Um9|Ycyk9+X$7&oAHXxB~s zz8;=9;q9muZ5|tD6iRXAkc|`_gdP@Y101d}Fw-$`S+V09_qj6K*{-Thm*k zm(G{A$xqDodpF``rKe}&*(Wz2bi4QAE_J8J?Om~nukY{+Luc1xJ9HTScvAK86T7-b zANMfx+tPl&yW`4;JKw$NoZ2*(D_d@TZR=rkvot9iRrJ!B$f<(b)vz?^)pB{e9WB~a zKjOZ{8k^Dl#iF=9@g=XmO08D3mJW3dGORO%cpdV1q5D-+)njc?a^-|-IWdD9#?|Rx zxu$m|^$fGehkw#-***o~B@^wQntSfS9_tJ@-OlQ!Ql~8q%v-GC_FJ~;{6i^++96~H6BY0m0fRX)!a2$G#bxk=-L@Z zc-ITKANbHu@Df~ys=t;s0xe7s&&eiwxynzwUpq+^LA|zxluas_n-iYV@;#q6Fg^9X z>U}|4%DazA4__>K`u0hshf^QzOTJ%xL0X}y65%82(o_kdFh;s|npV92o zH8n)QE!NxYGt9@!TjjCPWwbhid%$I@1eHO(L%YM}k*k}Y)7;`VGJWMaLNlSSzzf5q z&ai!Wl9~>yx|R87mhiH&kIDJ9^S$!+e5ckAe8b{*zA!@Yx#&NskI?{UHXfxaX2gg<1Z%v(#( zmYy!dyu!-!lO=yqBh2QX;cJ?;er0>eZdP>Q?GvRWG%RW{vu~s-@~nRZUebZa16BifmnWG;@jhp52Hm zk8KUJMPu2A%pv9nCW4s``>m&-nZF>ngw5k5p^`8dJSN)v*gx7v*;n(tgg`J|eWD%!IwP8hW2kaZpp%%bqc_B5A z&V>EmmdJk+`=0gY64*1Wo$bg;>|FK;tK)v)-f)9eiKIdP^qTkI%u zVuaXDtS+7xwhOz3IYOjh<{6=Y&*lrzQhMPLe}PZq^+GED7Vkd^ccJ&4Al?u?q!8GI zT_-JvjqNK^FIdn!3hT5xVH@}cEOI8py6a}xSUf|y!m4vDG$rlXY3!e@5p9;t?q?6O zYtS}std@Pt{J}J2ycikfc7-+jomfL4mJKJsPUBK&sc*yX@FGxYC2ZW0HBT4V`P%|J zf#;z)x`68{RuBrXwn1#yJ-|u>vEPk zN|Rv4`hc`S`UN(DwNeBuU$>Jwz}EE@$qOyq5!U`5U`47FD>EFa9m&10VCw;!r4#8h^jla6-v*23qhRm&9TiO1q}^e`G!`wo6;{so!6tGUXuk{g zYga>)Jss+JHoLB6AgQQQ(-sF2R7T1U=4B!);eOb z_E8G@5{r60u`1(-^{f=wCp?GsL9!}H*4K!2JF>2L7M3_SVTIu;{N7+~ECg%)jgf9^ zv_7#E)*kCRgOSbx*caZ3nh@K&dqL$b&=T&%^=J9D8TFox6{At*mL)rvTXX9LD}TdK z24Yon7}6L3zkXf zjqL4E_HU5&HD$G(thy^!%p5yw7{n!ZvJ?wl#I_8vJwsOb{ma+1>cb9NBT$s=SRi{a zh#kGUsDC3+pasf6c1YAfNyt8k8YmUHh*dRWTdh7ttkN~ce>b=^L%G`_9(f{h$i9h~ z@*NW~SdVLj806{*f0B9!gzb&G4})#HL3rwq-#)kofl6IKi~gX_h;l1<-H=`k%F+gL zIwJQ5$cLmsEaTP1Q~mPxYJuD$@LsEYC9!sSjx|slV&9UiE)q+0q#neEEZK$82=PhG z>g6#=9V(TlSR1q=X*Wh|kkIw9>KTf4J7R^-4*BCr5xa2p@oUH0X#}i*UWGNtIP|Cz znXGa)Ku8a)@?Di%!fq;|S^zA~&!jiAjnrAX@7U?W9eWqyx$vVjj$X`8r2<@ z5LGL65A}T}pZQvz#m}?fvsJa#<5vsgp@(ly>$nW`x4pEAx+VGubHp#0t*U!&eO)6A z(ds|3emILiA$e0YM_a@{#3po{)JdMeQYs7Emg|rIlUUL7qnb-$upd8Pwg@$(j^f`U zCH`iAYfZ3OY-8+a>t?M z!?0VKzvz1ONV=G+#wKz>>Js&0Z7=;^{dJe>u1R_?*WX-9b&It-R2r@qN^l*c&p&Li zDv;ZQa<*q1%eAG8Vnchd^|Gat^{H*BWr2CFwXY@1G^*@rY0Z+AMG-|?N@tb4D!N+G z8k(ZOjDmL;Q=YzmogSMLRWeza&!y|X^cd!|&^y6nvwJ`Ho`zm7zq)NOO!TPck>s)3 zFwXTK?MihbR!?`cH1|L^$?c}cQBSpZju+$cmy2C}m)oV9tXt~R%W%m1Z@&Y6_k1h) zxVxXhOimJd+4dWImW(fUFODm`o7?!q*bmWJ@7@=`-JNnJ>0v_g^E>fTk2gKK^lVVd z!4DFBIx9Wy}zK9W1kI&DBc&D%xn8 zFR#+|aQi7hAK@K6qiSZA{ZU-_!JvkIyiWs9)+@<7&u6UnVUH2+bzCL2Co@5KX#3f^ zkRQiy;2()BJyO16>tlLYdbQws_KeiSDa#UFUapC|^x$ADb@}kc%Xe2NS4zM6akl1O z)g}#v1|1vO8&7DwCCby&jfxico5YVkxrrZBOg?;1X(d*XS^Hw^z0$mbmjy1R%lVD$ zbr*lX=qjOgrqrri?Yk-sBd!GIdmPq(quZrFXW%^NdNY2mfky*Gejs&;G#D$>@` zw9I4@O2pN|KKp91o?^RuUy{gr<^htL1Av;p5+ zmE&1E)EFI8Hznp?_3(&^Ay2%1)1H;H@C9AZ)#7%;ds*=Huob}#y=NG@y5w=|`M#C} zTQ{kiWRbt3J}~}NU15Z2@5gny%hD#j{_e?x2Z49k>k}^RIJNJ*@L*NOw?!tthu@us zmJU_h?rO2Ng{9f_>IH6p2-VHkN|JH{)ABQ07lc@<@l8cv!7c_fw}r7~dkZd=^s>K` z`q1Zf5rJo`bcp^36x$auB)qQoNe+9M7(e=w+?v{=XS_xKE&gZy`~wnv#(Lywmv9m_ zL|!Y|g+am-z74;50)vis(i^<}^8KUSNkxM!^~7}S0&!&w)G-TNY%fYRvG@wu z%JAO5S1O4}iqupOh<;P)u6L^D6qRegXIWD*GIRAUS8vi=mC-4y;rgwt$}4_#g$9Gwe92Xd$tT^t6i;?Q!A|cjY=Ut zHMIMsZMM7Swx#Ecr}<~JKa(%Mkk(2Y&G+(tOdIqr|J8`N<9C0!yDc{5cJu2cSMNO1 zq@OTev3Ysysrjf~vyNWv+P1#c{79W2e5%ANR-UixZOtgG^?qc+mX~Eozo)Z> zV{AFn5!C?K?>#TNY~(Vfle8p1KnP--+D&0(c##454+0~Nb;;kkAMd|r}=hn%| z&ODl0Be`0V?$y*MJs(|)`|@G&{d%uP=dCWgD^i~0tDkE)qUn~V;~OupyF9X!+Y+%? zY3tm|DF_`hnKQ5| zvdtJ@+O6b5VV{EWdB5g-m-#X+2 zV+`MLdc7V|JG}a+Pm9M&mt{RjJ^B86*46x^(ihg|@)6ZM*F@KK+GFfe?gY0;&as(< z1ZY6CH*Wi2ox+Ovi(hc@tz2^H*tURO2yh`u=*Z3{> z{@pc;$rsM^=j>kAo2Gk~LhA%uUE64Dl&MXL{BeI?Zf1wnk#G7Xop^QgY1jC@PtxO# zJ{|t{Sbk`kBHTjg}l8Cd+%oM}HKol<$| zFK9=oH!-8-)8YdD4*!$r#r&;lKtvM z+ZrFy{~!OcesjD7yrvjV>8q+=u|KhAxHRrM_m+4KJ6Z*2)@LvWgkzmmT?Aba3s+`a<2`BJ(_Ln&0HL z#%nqL>D9A06{ePinrn)q)x2)3{)8ruok97_Df~*C&OR0USI!t>y{h>22pt)5Gjw~v zG2fwH-?`RMEn)0jpvGG}Ot;bS&Z~*{QLlmS?F=z4FSISO5_Fu&XLoV~xr^LY^=nN% z)h((-(A$St<`mb@pY`Fz2jA>*8PhViWHriKn6WDD=-Zabf4tluU-t$5uF3nL%p~I< z8gDPNZ|87Twfj|XR;K)BU?qE`99&wxux3fG(($FW%(+s4%0pvSKhq>?kFXP^7~wDe zZ~Gd418w9E=>~ZY3^*IKIdE$5$sj}E5wEo_nfg%I5w3r0pX%?pZ}H4C>~Wpsc2)nE zb~JmLd5-UDBRfN7S3gy4#=3Z4Hch&3*Vww1)hha-V03{{Fh4&hZ`DVtczbc(!e0xd zyvPsl(hbQIU;XmD$&<#<4ki7bRw?&;a|iB@VXt4!kSihIg`Nx==Ke~3nm#5@w?2VB zu$Zq-UsKb%G?zJUv2N3Kn^c$SI-+EYvo5q6ZKc8_no%2F8hc#we&uWT4-Az3`uXnk ztmHY?{VTUS`l&7*T?_SFT^i{7=&I;9>#yiTw1Q@$cAC1CsvUQj-YG>1efc`px5lxi z>&E4#*QP$^c*{MDx23@JvuSEshvH#{>W{WueQr)xkIcL854~gF8sEKoACoaL(^zoJ zY_pCQ&Z;T*dmdX1k34(&gm|rS>#eP$>93i>-j(}f$4@%jmF>@ka~bNfni;Am+yqsK zYAV-{GqP^#4Vo>wJeR4i@g6n3w|hVKzU}?3*JjUO4F9++)j!mB)9hE9ROeLr>blx5 zv{f~8G{0zWL!a@O(nYxS{8*1h&Ewtcn`o83~^YBilGt7H7WY;I}$ z(&Xas;#Gx0;f#+|?(NLQS-rBq&2IW(VP0BEJ_u%Smv3GEbgS=m+xv!( z&TG73q2X)S+PXkhCGKn0b2f|F4o&Y+wWQ{;%VxHQQO{QK%sP6q)I~nRFR`4kG_cmO zx>(=YI@<3FaYDXry3k50;>X)3TD#l#TAx@pSjJewEsf06%AS|fkc2%7e<(cm@onLX zqA8H!i%U9LA}ky1wS}7E0qi=`QU|a*vLkd_<0&(p&rZOoFdS>euhk!+>m1JotJ2hu zxZ`Yh)=Sk;6UYT(M^-m(9=DVJTh)VIp*C`Z)u+_It0$_D=(ec$sDst@xIER@Twl7KO4g>d+qP+q2g(K6!x{XwQm%E zmbwa7DTK-qPhofJWBEAr?i<*R)F5#Ic9rhHe%B>ZCS6~Ol&^6G@~^0If~;cJW1r^= zwikO68h~)EmaR=Os_CkM+)wOn<_!A>^_IS=)-nvUk4=&fsZKLC`YqQLUs3|}Mth+{ zyFj}#sr++#lRN<0nHta_3F2kuusnwPlj}!~2bKR8E%Z4_Erv)RX+8Ue)R9V%rpa@u zTnVzW{F^?K(ut#m4Pp-|SPJHs(|+PMb~^u5K1d1DVeF9^Dc=yisb+F5x*K-fb;Zt% z7D6L2gXttsr+d(%SeqKaj1Vy2WxHX&XNhovo-Q3{cS_Hs@$BC`!#t3WGiRwD@-o(! zX~Wc~w=#X1aIR3^th$FV@$^k*8r6j1p&4nW_LUFNiOfZ6JJSsF=do-%>aJ{He!;%w zXz0}@h?&@nI+}e&y^tcfWWiHC(tbl;udO4U<6MMps&AkNn8Y5Ik8!=ls&WC-h&>@T zrJhk2I8k^aZD-=y>-_hc$U=qc%%zLag#?lY4F0v82jQ(^#?1J^DPRldIuQ^Y7 zyKs?PN1dd{h?^-LG?1>0Khs#AfiJwP(2}-hUm^B<`5$VeT#I{x{WWVCj$ObUm#5O< zqLIy@S4i)$+u4unB8XVGy+!>hN6>0|l(bnIO0{6mVrO#^?IRCm^C(7oM!8^b^A7n- z2D_CJYXSUHm}T-fxs_Ct{!y6CjN+_P52iXZnzD-;DwkU;O=c#DsZ<@NJ#?oWc2m=A zveb>TsSYr|$!&!~aXojInM;jhPC=Vmi5+jhPx~^%n48QY+YM$p^BB9YZ!!f^EOvws z1{W<6_0$1o5+&1@`0Lyy>{;J0--n)SJ-w4!B5jp_0=0_yO>%8!1~kNB@;{uv_=2j> zPC?F+v`)2OUMa4Z^5r>P0Fxwj$6oD6*t6e2RHI?2&Z;-x} z_e(!8C)r)pQ1OYB&yAtl3U2fhPLOWM%>*0c!#+b9E=U)-B)J(Ch5jb89i&~7t5}tK z$uy^bg0}glbeU<5U9C-}cF>x{)89dd)`I#-4WYx>_u>-TMBCZlQ4`o z!l5nw5&OmuNeAekAsYtCd-(&*HT65&ZqbLA*a@n}yjJSUrZ5pyPjSE0OOwF4rb~mwWbT?YTpUmBVNX$7p{1P9OrZv2 z_xmup6D=^Cq-1KGd`L=^x-rF6OL;P#B#&o{#X@;4Rvjd2fILyILr;Y+^dG7NyN21T z3KVxp*SHT%Rcan{Lwv@4D`{;mXv;lpJoQo>N}+#C7p2DNYa6Ih^hL@Ga`rDs&N6z1 zz%u>NrcLQIsWV*%rvcv6*P)sIR-8(QF+KTk`VEsN&%$1v&Rj6_O6(%H6yw+%%s0XX ziD9Nl3#C}>ed$N75$14rsB|${PL^h1C8@XcSYFC5q7Gn}bRPOsuyjoLnXN0?MR#r^ zHG{4}S+IU6%Gc=H=)D7^+R_8|o^VhYMY&1Yau)%dN}HrpLZ;M>9?aB}`Y}z#_o9a7 zsmrU6x(c?mA~1>9=HzS5PHuiRf8C&^qiR$97A4big##b!bjU4_n+2J>E!lB>l= z*#G}T+9wCg-NkwII-ErKo~kQvr(&fVGB{QqPyH;Y)d2BLqw1?HZzpHL+fN;tYti8 zX3DkY`EqsKPlIkg)NEQSx1bu)yTxyW_u^!!iTGX!5!*>WO0U2#kyuYr%Vm%mEyOF} z^$XG~aU4!jJe1}_m#vlBiM#A|gHTj2!9 zb@1g#W*F6!J}TR!T)B$mhZ77LvW{+weW$mmNVzh-K>mSR51#8q#mcv!f2k`4G11gY zx&dllmA-`?+OCpGYJm~92fd$3r_vx3V(1A>OYAeN2YUEZlcAeVl*UlKV%Ej^qLTco*uUbRKjmlD-lrL;H1I{8g~=@9i`A8T<+RGkYALC43Mj@tgQ5 z_SLo&n_$^t-D_=ZU2a`x`OVVLa?d)$y52g_y2Dxuozqk6IBP3wV{@qaTk}=3-Mrk| zl^-p}O5b5zc!D)Z54My&!P(h`+*O>(=%ZeWd1jz$m3q4BF?)-PQ6EP6i!c_PXT6vw zv`niR6Z+SmSP9%rCDSKq72ATn#eSiRQ@vB~SC7)P)K$}&wRTN}wy&qo#)Tj%J}IU!AOe zr0U3puo%bW72*KFj~|Ftd@uWY`vm?SW|G$>4W*|KV|}wG^A}d)XL7r_SXCd52UdHk z>VDD<&^6JG(H_+_)6`clRGs7QbH8!pxD8m5ACC3*4Di8J?i)_WO<}9C8g>u!i0QH#Yu#bI#?TNLwb)vO~HPgacYgz|mwLQ`Dm*opf zx;flD(=@>Jlc|R3jWN`;#uQ^(ZA>paRu)oLsdQ`!UAnX^!1Rl`qiv2@i+;e(&_wIM za;xDU;T~$};}+w#+vQK)QB6Zls3t}Ihni8>S5HtsQ;p+>GixY=Tq3NttF3#@DFcZZ0uU+sakAf)dY2+x`QTN(?J`h&CysiUA3LHC7NlPBWibbZVNw+ByYDz*BM6ct89+cJNt7G}0TC1d$w3StNK{?}K?wqi zfFdX$Ny$h~g5)F!f*?_nAUSTDna<~~?K9V+@Bf_h?R>g-uG#MCuCA*7b#=wNo?0W_ zmzSvh^sfTNgWH0ig$|(|`CV{O@SEU&f}MiZgI$89LRCVWg2_Q!FQ+fUJddmH68n^y z75g#rOxCo_VVOHKFJ{)tY8KfTy&UUcoHo8PpRmeVKbyCVD#oM6px9y%c_q>}qC~c2 z-N@{fIV#i3$V$JE_HCM#b|C$H#x7*7ReOas(wbwnK^{}tJ?Y(+ z52$1GBEhYp!U+`#>gf9DU(v;}MMe*Ek~PyF?Q5}jns4UHE9cSsLSq`m`nS*cMko(5tOcZt?R~Z#Odc~_t0{W?;72l)u{=sNCo=RGJeu0k2p&c1AIu})j} zt+UoR+jCH7L(8PKekwRW+%NHdQbJ0rlxxX$(vHMB38|qw`k!iLrJ(ewJI9IIAJ~s# z6zZs5-09@ZbyW8qcMV$JPoO+IjG3NoFeh;wM(B%6yD*1xh`ShhNQPArZ`DqttZ^pR z9G{ff#^?v=cYi9nHnKAEdE~pu=tx?m5(qbpEr^|oU5eF&!f&Z@$#~DKW!@>iL=aX&$ z)MV#E^N@ua#~CObg3^Cb9tV_~>R9cpULNn${BY-lxd~ko@+XW7cMY9@mb;Ut;Vo(; z_4dkv_*C};_fK~wM!IW)N@KLW0%+r;%a1Ba$cg_(8#U8=5pxn2I=L}>>Qy@i&CZk7 z6IN|YvfeQrqlj6}yl?b2#u}x}b>?^GFtdcU&njfEvOcgDSr*1`|1>``2SOEez$jzP zis`ZW(cRIfWBp@a8fEO}-fDS^dK+_HhM?cKP0~|IUSf%aqX82&)}8VSZ-vv^seoA# zgE8Z?IqptIi{OmA!|mZ-bM8Am+&ieX)|WN4npRjZ6v&1Cy@}{GdseHZE|iye`P?sX zebk->_1IBox*Ky7p$|_|DxO9S!OC^&Gr!cwABV9|E;Wu z^&ILRUpQ^h@~H0SakqhcPpgDk#P~i|D^?)38``iQ)?oWnrzPI)MVO7OqtBp>)LEXP zR8qIAi}b$1q)_kRk-*+Sy+BZ_D6hpl;;|Ut>VR@`w%ZOZ%FEt$=~KDB{3Y}anV9{& z5h|IeH(L5u{#|)Z?V$EiRV_>1tMroRN`0gq-ct8rsD&CsV|3Z;Bh{8ov;rI`d_I<| zp_gJU6gXtFLMA>RACB<@zb~+3nL;qApWM-YG3X8)b}iRyr=9Q2MLyt8b{? z)hEE?6$h?XeoY31ZXN}BSS+Cz(Kchn6~ z%neei$nBuo`ctYSPmte`_krQFa%bpQCVD@+7f}+{k{*_;Da+Nc-cNT>6I`v$Qj4P8 zACH>0fjaSIbq;C^UG#y_7ba*Im0!@dj3F19hbQ+5hx9+Q zKJIMt&O@1{LBq8Yd3<}fxN`!ExB+%cJ1=TuI^M@bq>XZDOVo7B!=T3klw!xwJ6P5! z;5PJ{g48KIv+hn_Cm-H}AsGGbg|^{J^;xYm)QRb8HMAs)O9yfNTd3o&^Oj@&dRh4` z+=nSQM#>ZYeW^%c@1kDyrm`8T!&8{uogo!QZ$fSP5BY#{R{d1Vt!AhpyL-%i$=iJcann}PZn=LhFkXO{Cl-px+P*AroDAg}i;Qlo?(jIqeQpK`tv{RHk*uemf7TChW) zdkEJ)iJ6zbxK8bOL@Ymp0(dSOhg@E2zoJAa+6zHxzIA^U$%$&?5@#Fjt{Pl+a?U51nE&#NidV zDTr2h6XYD7p%EOOt!=D>YpO!;7ld*Ty(P#O@D`&LH4#0*Gw@^vBL4q@eT3#XE7W)m zpybowJ|8romr#0F$=1Ksfu61#zLn6D7nVCrSRN4;T`uEUT|mq97~0jhpo$cBRnky$ zXmBTVi51W%5JC-N2UKk*p>?~6esV>+huGhR4;PTb2wPhYu1WWj;ZoQ`xdER=$+5-T zh3$51kHd8)j-G)JyI>tx%4^I7nB3u1m9*ImRt+=82PaJ3y4HV(nJ>xff2@}gY0n_IBgatX0Mj?mBG zu7$-F5g%bmN7&rCf-A4Wt+1t41nJ~yur5^!DoSA$Nhmah?WIJVFN=?`*i;p@D50*b1tnt>=n#ccvI;^G_O5(eSHAkP zQnnJY3ET@crO+lez(*(`{g1GbQW|MQ*fc2*H+8em*FqW+YD}S8EP@nL9($FL9;zb5 z^0-3SV)1QD2~DT4U?u(wn^A?4stRE%EI}26ua9H<7>*Xp4#T%(RVq84!eUiv>DrQ-?-?}xZ9hF|`p6zB=36}yR*+8Ox3D5~5@&?HBw^j?IQEYtt@&1!P9w&buzv;HGuhuW_$`hJ zt4zWQlYdv@m+;^B=QP38zO5! z-#f6s8?Jq;PP`*u)MVkj+inr0 zPRnbAce9yv+IvcFt|qFxVCi6-^cbv_j8}G}muweW9s`s$cyG@_X*?WbJ?BuyRF)S? zJ6+4|FZD*7qZ@4X{N=QU4!@0;4`W)55JrNO>h7}x&U&X4Y5+Pc_gr%ac@w-b7-wB0 zjZm7(CsAj374bOk%yTQCw$cORKgHZNPC3;2es)?o`JFeMU1)<=a#K+Ic+-1M+V1s7 zjb@bB6M3yB9mo6814>oZnPo?u0d5)e2>%4jB?GOu&Bx7C<^i-lentOVOY5LH*ji|} zL%rz|eBEcCLQmWxyPR_Xy`fvsANnI&s1ekSesN5ksqgGX&AI^EHu+H-{M7m0y@4k& z0IB8@#yA}LlCn(sNq!Bp1J28%6diq{rPbxI_@JZxRY=_rJ?|y;0>%=@BA;&{-9$Zl z55@_h97Y{)m~sJqm1`9+Rzj%N^^h|$@4%GuVcai>Hd{rsC;Gu}OIqd4M!n+`XAIOq zzoT~{&F=3sbbmwbD(23{Gb@2HqH<_uKWQH{XQ6%H7H!v2W=pH1{XFiXwKc`Oi@7V$ z8Ka}2*vrOUW2hk;&!C?vWV~-wMB8_?Gf9SOBJ@~jZeV6$R$#xjRQW>g4ejtP>8?`} zy_65xFWQrxx!x1XpHR@u4*ZOoYd`b=uEX4#BB&9+=oNHdw_8GcJON`@1LYKD0NS_j zYkg5;-hdf7xzvJcenpq2xPPE`bU125Iz~G)^q!aplMtF4UKIW-^aA=C-wd2mx5`tX zRNv)Qmr9^VdWbqqKOFom^hKyvFgY+@%aDI}v#kG^gUqXDTV%hMT@VZ`sfE%Uyd~`= z^+WkR@0!!VZf{PFjf*+P3w9NR*%Lhiq9bW3 zuv*mKId5NqdU0Ila9EZ&XkK+Y>Qd6nsq2#73s(!B)Gx~Q@J8;&JJ{6CZ7+mcMm1MD zr{q0au~6&8uaZ7WXdC=VJEFWM*Fp=ilQ-JwffTsTdg$1R!9}6!q2n;hY3aXet}yjv^|>0*QLFv^>suGe~r^0 zZO9+Zs8QP-W=(Vkd(TMqP-lD>qf;X=izeCm##|ShAN#{Bj<#n5M@QST0@Uk|DT~}K z#(MOC`~|xd*CKz!9D5e}5#Diknk^#rGv3Y2it27^pnmdOsW+3&@B+Op#>{WoMeL7Z z@NJM&9(~WVOwC^5?N-MI$0YnGX=75=gqHeP=?7GsM5tRr-DO6--a)RONW09J`-4}ZpKK|IMfOXc+aBG@0#?vQeHc$ZPxecSG5u9MtO$! zxMQ1zjIOcUvCHNXXC!j6*4h#K?LcHC?|H=#PHxetKB{K;0vi!ie@n`z@<6`Usl2?d2rxb^YzY z*MZ!Dj(T}*g7Uu99lc^6tX940q`GaSLrR*qC@?hKFu7Oi#-v(7OX(!H_X;^z&38)iIaTCsY7>l;Oba&CLr@Ivj$Vuo zGz>EV_QE1iAe@)}bW0%D?x_@o=EsswSOZ{AcTuLBSuJ|OsxE)1@6%zfvbV;=-)r0r)t|V zoA0bT1@_v6oy*f`?XHsgDjM>_AGDe3O?eS&k@qlH#6yetY1EikV9cwoQ_O9R-k{&y zOm7Kh*43Btx$~^I%pTS~yRNgtZtZ;M_JHyzpEJNdU^jK^qJ7l_@79diZ?SW*xzpTy z*SU+4-b1i-_PO<}b;HbO=~k+>9Q%ums*&`xdg+s*6TOdutCOD0wIz3r)S{tnZma0} zj8p05GP_1knGL-a%G3ImK#5?aB!mTRa3wb^Q>G{Nm>{T{0l zT^6ljOv95pgc338O_y)V3zT_sKB=>_&Fm08n^`d{5@~I2v3s~Bp^xfkRLHz=@6GhQ z&a2@!bCu5BF;~mfC5c;jR^Cjv<~lpP-pae$Y5i1SYoMCm zPkjzETZ&>n@J%yh{bmioJcm8fQ8hREeZLGg2_^=LXp@wvG}1ffegb=jPeY~pP{0eG z2>lf7sE<+FVDzdJdcDp$*D#L%n9@jnQB(C{`bO<3*k3zlS;j!4k~z;B?xcHNm09Su zn~Txp&(M?6$LVbUVLfVJvtG7lnD4@V$W4s&SCo%Sf4J-Ioz^_4e|uVc?dfiA%wKA4 zCdIlVZ@Cid1?ytxNu&dy)pVr#;UZ1=?yb;866cp*~xt@bz)Mz@JxM^HzHOl z6PC(iA?tf8bgyTzVoy<>H;OJH=@kgqEH zQ8Mh*>jqy7{T@t(MY96>9Br%ofjb8Mq4!`juq*nEQZ+fS8>3dQsQKi_rH0-|P~rS( z95wgZPr6s#9bPl^p1-VUay2gl{il_k=P`D(!CqvY!EEL`#!=&@xx>AnRM2N=1LR&% zofnIi%u3JvF1p+vC0lA$C9k*Hdedl*F@Yhmyw=O^a@0r@oV{kH=;_Q?GeeQ1QP1k8 zrYBmdSCYN(^TCg_Bi;xje^%G5)L1w3s5#0S=?2x3!OgIO)hDzpR3bD|e^##I6t=FK zdok{C*IZ&B^sXvB=g8jJkv|0u#`~dp7sDqyd2f-p$Ci`-ks!&UhhK#QxU%TCE(c5w4q% zoX`}O?AitPs~f!DRs~~kY=qItJZ=}0J*|3hC&pVh=}#+d-7?ldBcHL{7--E8Qw`eRaQfI8ZWtBs?&54hpy7Qdx|#ltlf0 zyu3i$8t5IA12yzat%^1ZTA)^LMW-+N;4#YKo{+XE$JLlxNu2~0$Rzhu=d#_~o@!Tj znmZ+&PwY0VqEQ((+|kw%L`iBe7P} z+llQ%YgNtLhn%~JJq}9Rwd%0Y-w8h^)=OL(?h=?Mqvz9zM5M@gbiS=~i^+LVTAG+6 za2l$?FPzS1x!5OBBihE8Z#@CKEYCrCxYcdzHgYOJxqH+sYge@&Hv1bFjGbl$dyP{8 zy_OfeW^O~fn_1J?7aMKNHGiynV~pcp680*d4b~9)y2gnlQhdbtBrq{z;P6@~S6~L$l}$BgNQnEVh1hmU>lTgZFFcSEZJ| zSRbZ+rR~&4t5u~Fc1b%7J;NO7g0ffJrM1(FXn8R&wF7!_2cm8Jn>-M`=mn(=s7oL5 zipgKAPib|uJX$+-wLIHv=j_H<)kJ%syIQ`f2NRx4u9VUzWmQtkaBHoCSJwU+b+8>~ zHRlugs$M@~M)Jv&Tgm+rQuKSCgOYi3R`;v|(KSXd=Lh+1tz+QDz%hNawhT&@B&Vk} z)Ob1iS+ubEn^Od{l^ZBs_2z-0>LmBDaVl$n`oOf+8Nvqf|$png8-R!2U32J=iRLZ7}lFdgmA8~SMI`Zn1E(X(I6x#;ea8mPBm_ajSR z8n_hP5xNvq^rFfP=s#7b2rSeVmd0R4@l3r%a8r0^!Y>Ij+8*Ef&Zh{7$c~eI+Yyn)#?vDrQC+M@O5wTOMlDKI*UfZM_=i`R#GGSeSxgoQ-{I zwUM4tKlLiY>c;P=Puw?uu|nQH^y74Pni$(6Sy`Q8jsumix=cSFC>dBPt%{w>u+l%x zD3Nh9qkd$r*<5O_l?l{D9T4^aL)(@9&SJBbc^GxTSB>)aZnPPSs%NwV0X3WyUaPm4 zJ!hXa-8dB;80`a%V&%|hiGkE0w#he{_rp0G3co~nD3tfsM-GN#{~e{Wi5qBS2n@v(u^;cvpV^i1!d-QM~zdOLk> z+OCYEvC+^B&r_ccObxt-G_uCL6sr`CMux=>Sid=o-L7ssdy_FH_M-Vc^v`Xzw7@5U zdCG44s8Is+4xl6$FCSH>sLj1QW_xp?okyyzO~;&^T&Xp}&82|(S)@}|uILMPs=hYq zREm|9lAs5cD;v;ZkO2^TDEp*i?2p`^^P1>(*k#NUA-$^aU3Hpaw2O` zY&rCNyWKQ|CFX)(wAMkV{Vh`SL~DR`$a)5K^T(Zsy~=VJbM0T0s@mzX4OwQ^L$LwY z-|kW9rJvNhs3oNYrcl8C1tzy`k!4ZJK-^%Gnhd zy{hb-m%j^cgN|oXsBGYp`l(z93SS%jW{aH>?pVk4TI*XAKTl0h9hqDq!BVf=ouaQq zE1B;(-Q*=mJ+lMr)y8r=s10;wjp?rz9X??Rvv}JUixy@ZF9l@A!AJvjpSbv-Q zoomt;&|aTJA4FN_HS<7pW^^yw;b)xPUN4L+2b4$L3zlJK89Ok81^ckn-4(5c{z`4P zsny(E15Hp$Y`sy>PJ;ems`rS}QJti%S6@>jQgf#~tifH6-H*PCy4nJS_qn4~-EV$2qQ0h%8h+J`~*9V{*Ns$LX5W51$yo+%ZFa$*du<>=OrMt`%J zGu5jK8@Fk}TEP)|E4hOG7y9ECLhD%Da_yg?3UTa4PNLTV<8mi4{@NMJ_U_&^t1h1L zY^$LSErwJZ{kMNQ^Xxp%Y}Begbs9ozIshd`edt0P+r#a{PFwd7816xPVTe@#YN}?= zo0#8lAT%LdR3GbJHlH>&Mt_a|YC#{bp3}b!bitiuXhZa^dQ<(lwnJIru8p0@sun$N zyzKt1E)MTcxtMY}(Fi@N8&YNGj9I}d>4ecwTNX2-Kat14Ud>*0H57nV6Aspd)} zV$O$dxv@M=8ygxM?597XobY~ij+!6FhQ*E=xy)Q|O{fa0rZKD zQMxN{Dz`9msHyRy(cFH{-fGo!c1b7HK3ax05Sp|u!5XlKu0tbG1J)+eyxH=0b(!XA zM=|z(MVX?mR0koQ?6v|cIY*R}!G__F(E=P3D6bZjF4;G&M$ilGvpz-}?4;Zi zqxu$lX-lhB1Fb??sCPB=b{T~tceAWmO{c8XSf3MmEA+Seu6q@^dL3hs`I4hSmGrS& z#JOj~js;W-xxw!trI|d(Erk@k0y?hJhGV?qEJGXcdu6Iz$(l{>FIs#oVBK!BW+1egU0lRd=Iv53ZSMl&Tj3VLTNcn`CVGzY&H)X zh0O1*rEYg+94v^mgDvyf&~Qz0PC5O(J<>k3Lk`;`k^(wk`H+4?mc%gMkFwE39}cfsQaK#y{r6=y3T5M z7{>qiqomL=TJ;TdR>hY|E;^uw`n~lzH2$Axy|A#%S#`9$8c)0kp4^8}oc!Qafz7BFU^!&D`wF1h=N8PU8EohBArGWMj+AKZQ6wI~QYIimwv2W~EQczo}FVNSj z1?60@$e~CTlo3!z+>}-z&&&f&UvaDlFdHqV0iomoB(P2$^H)o zq$bo?FUf7tk9thPly7M$`nRuR^kKPl31hh$^ipCB?KDQJ&%=844SBV)1QzqRUva%~}ldoZQhN9>VT4d+4JK8_FMdh_n*BDAYjFgp^Yam7; z?}VG~HuY{|7GgQ*#in5%Ob3ko!mvGNxI8C%{4i$R1MZ$cOy|3`aUYm3CYQv>#T(Kh zZ@Jshebp_Ce&6aC3ttNj*@vjdR)>z_d-#)A>H-aAW2mn%uL6pXI#B01-dAo3JRQ$T zz&KN5l=lVXjdCfho)D1hKvT06nv^ZhBxsK2xv!vYxCZkWT1)%AzR&}XfF-gi?lTy- z+X!9gO!U(om6s~7pr>eqI#~YPIfs(}bMy$W!937nk}bDT<}0pR3VN*r>hIuaA&_egf?Z0v=BD9DnO4{0d5o}f^n^-(Bv3Uj{XAcWVtY# zW2iJ33M=$ID=)~?V518&GB8%Q1oYo_3&1+rN+@tnNeAQ%X!gg$RX6!_^l(*lTS9xY z6fKkIu_{R;j8Rlo7T^hdjhM`LZ#j~?0_n0mGiPDU-5Rkm7p7a32)U;m~(ao_TI`sx6&KxMNfVXHBD%npmP|6 zk*qM@wKt)Fnul>=u`bw1jD`-z`zKP|)<_HO7F=As8O zRsI3-PDfnc!kF#r(0TU8*&mS#{)95DIIL0~hQD>OzDjS5R5yZBq!L`OMGpTk(pWCM zO)H^lnt`J|F-rU#!pQ~AM{$JG1v$ums4-VcX;LNV20MUG0i@F6sIy#x)xj}PxCrgU zUGH(cjisec-WZGvcJX3Ry^YHrbAK9RW%*B%d(6ODxSimMI^g3c({t=AXmqHF9@`PlpLh%i>C{VE?j#%LYnz03V|MnyO z_JVrt6~yT@6r`_!;xgpZt1%Kj0;}sBLL8bRB!hfwq>iPC$wy#cAKI<%U{)2+@C=?_5L%^WxQ|W+2LMR)mNGIX{4~X3Xs2J`d zB|Z+-&OM~V=aI+cgC3~^?(97%+|Fmu=t0;>Nd{^&18_|#++TCt_eIcZiPSd%y)zDu z3dK_b-ZWt=Py(mYaHAn!V&$mEp=9cZvSxwwGxCqW;qQLjPfx^KXokuo+`dQ$t?-_S zmEgqcaUm$_Uc{TL!OtS#xfc4FM8vE#eq**K;vqDVyP)&YaLqaNI37hm(K4iwjd(sq zz@{gB>;S)YqJdsO5$A!Aw8}@o?+o>)CK=RAW;N&AXe-WehOV(BJMVd*iOt&RU5!YM^4-V zbPD47bmSdp5!ZEau?4Z6g;aL}yl%q>q1ao8>n7mb31~4=kRv^Za7rSKQg|+fKuYMD zPT~${BYkbhXEWZ_3rI^seI!WT!xPLXo!>3LK#u68fDZxMm67(;>LK zCHQp-OdbQ%6r_+FxQG2n74s4P0mR@mVkLAi3f}j7NV#c<`$2@g8|Q>&yoI2D2Cl@~ zW)~2`4#Z|H6gxsawFRH8_-q2NL)q+w4`L-iv6A0yJWHVz6t4Cmc9(H11SO7GGfb$% zGU1<49_@yIixAFQxVi`CwO~WA6_}KSs}l(CD40oL(*kTr%P>$s16_@p2_p`Ofy%X@e}MUV#aA|@5G zb#_m~?>2ZFpTTo}9BCs9SLR2Gua5hC9`{)n$HnTCzPeA?TNEpzg>Z+N@J%Ro65(G- zq;TO*Xa>bflvNPtMtDD8LfUMNeW6b+g|otfVlAi$YvHcMeF=M%&ETpH?))Xhq7Hn? z4zpkCohKhA1Wzd6pEj%$_g(_Bd71}{D z^Fib)MG%fyg;T5`D%Plz@N|V`RPhx2b@Rk}ul`jc9c%DWtdS>lfW9l?w^%<>tlcNp zO%yskU#WKi=}4^JC$x#7?Cb6#tsg*|7V5I&a4FX26RS~P#}%iuuNUvIzq+4TjZv)T zC)S8d%Z`h*0g-NmYt;_2VYCMZ^} z6f0bc6&bH&lN4Kj?M$(TreNzo1E2N;(XkH|0xNo z;{28Dt%&J4gfHBP|3ZZ+beuwoDb$>Py!_Q9e}h_NZ{>KRxlLi1>=Mg84<+G4R*E^r;BKzVcPvyPzlPQ-XrnieK>*FI@TO#aW?= z6=_gx1xvwCC|!@CWDqNQiq#v%+MNfmFFxY8zjmllmI__#c9apK9PR_yuvYf*}?&Hk*$xpnxpD*L+( zYO-bcEX7A`#qZ_WZdYZyS_gN+rMTL^Mr?(z!tad;(GN$&a8vgFy6mk8-{14yiTKB3 z7oLw!S`C?1|q#Td^;6#Nr$O z2=YQ3EVl8LS;bbYz#3O23)QmtiZ{Xkh_^z#5pmVEP!S7VwRrX-Es3vqKjLXiybB^# ziWKapQ<2Vd@(zDhU6C(|bShN;VzpqA>*nM|LTxU}5EK>2)5XVM!CTBx5UY5LHN3?~ ztd}abLg(%u5nG{l_qRg(F215v^H=#6*CeAH^uMA+6m^35YTaVpRAK8`tidWQHH#IN z{T00ZwY!BPU)XXMihX}&XHh5hEkFCKgvZwo_t#wZ<0N7$N;R=2xwsd96?Rbqi2I9| zeB!r@GE@8(WuaeE_-^8E#jp6jxHjj{C*=PZtVBrh@I;7$qxkwU$l3SL#0kf*iM#UI z``5*N_3yxcF1|bQKgVx>b!q=d+^xU1vx-y!&6{PU>K()TxhRZb}JvpIg}gcOghaQ)9Uv*T{!cO_i-K0WxH;-TfZ&I!%G z!oi_WZN)tHk!fbN;^XdroS8&_(?EoNYWV!d2Y2_%`QAPKfb6KYnqSIbp<)Jb0}L z&Cfjq)f~5?r{TfB2jd{zYo9RE)Y<32oi zXa79*PoH!A6<5Sd4RM8V=ZEF*{m(weUH$Ip=cr;^*S7#TjudemuwJfA9U<-T!pW|8(rXefsy2 z_*FR}#E<;9%m34v_&vnW|2Mb)>4@HC8&9^8NM&G4`I=h=Vy=l^=pg}?vrBM-(belAWf{`*0n|M_$tJi~nwAOR8}0TLhq z5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH* zAOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8} z0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq z5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH* zAOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8} z0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq z5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH* zAOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8} z0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq z5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH* zAOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8} z0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq z5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH* zAOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8} z0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq z5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH* zAOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8} z0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq z5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH* zAOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8} z0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq z5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH* zAOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8} z0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq z5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH* zAOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8} z0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq z5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH* zAOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8} z0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq z5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq68Qf| G;C}%>>^FS? literal 0 HcmV?d00001 diff --git a/tests/core/sched/test_omni_ar_scheduler_streaming.py b/tests/core/sched/test_omni_ar_scheduler_streaming.py index 4ae4f7684..ca80323a7 100644 --- a/tests/core/sched/test_omni_ar_scheduler_streaming.py +++ b/tests/core/sched/test_omni_ar_scheduler_streaming.py @@ -262,9 +262,52 @@ def test_explicit_streaming_payload_replaces_placeholder_prompt() -> None: "tts_token_ids": [10, 20], "meta": {"replace_streaming_prompt": True}, } + update.model_intermediate_buffer = { + "ids": {"tts": [41, 42, 99]}, + "meta": {"turn_eos_token_id": 99}, + } sched._update_request_as_session(session, update) assert session.prompt_token_ids == [10, 20] assert session.additional_information == update.additional_information + assert session.model_intermediate_buffer == { + "ids": {"tts": [41, 42, 99]}, + "meta": {"turn_eos_token_id": 99}, + } + assert session.status == RequestStatus.WAITING + + +def test_model_intermediate_streaming_payload_replaces_computed_prompt() -> None: + sched = _make_scheduler(stage_id=1) + sched.chunk_transfer_adapter = SimpleNamespace( + receives_chunks=False, + segment_finished_requests=set(), + ) + session = _make_request() + session.status = RequestStatus.WAITING_FOR_STREAMING_REQ + session.prompt_token_ids = [0] * 59 + session._all_token_ids.clear() + session._all_token_ids.extend(session.prompt_token_ids) + session.num_prompt_tokens = 59 + session.num_computed_tokens = 59 + update = _make_update([0] * 10) + update.additional_information = None + update.model_intermediate_buffer = { + "ids": {"tts": list(range(8))}, + "hidden_states": {"tts": [[0.0]] * 8}, + "meta": { + "replace_streaming_prompt": True, + "next_stage_prompt_len": 10, + }, + } + + sched._update_request_as_session(session, update) + + assert session.prompt_token_ids == [0] * 10 + assert list(session._all_token_ids) == [0] * 10 + assert session.num_prompt_tokens == 10 + assert session.num_computed_tokens == 0 + assert session.additional_information is None + assert session.model_intermediate_buffer == update.model_intermediate_buffer assert session.status == RequestStatus.WAITING diff --git a/tests/core/sched/test_omni_generation_scheduler_update_session.py b/tests/core/sched/test_omni_generation_scheduler_update_session.py index 28083349a..5cd829925 100644 --- a/tests/core/sched/test_omni_generation_scheduler_update_session.py +++ b/tests/core/sched/test_omni_generation_scheduler_update_session.py @@ -54,6 +54,18 @@ class _SchedulerStub(OmniGenerationScheduler): raise AssertionError("unexpected enqueue for skipped_waiting miss") +class _AsyncChunkStopSchedulerStub(OmniGenerationScheduler): + def __init__(self) -> None: + self.num_waiting_for_streaming_input = 0 + self.chunk_transfer_adapter = SimpleNamespace(receives_chunks=True) + self.enqueued: list[Request] = [] + self.enqueued_statuses: list[RequestStatus] = [] + + def _enqueue_waiting_request(self, session: Request) -> None: + self.enqueued.append(session) + self.enqueued_statuses.append(session.status) + + def _make_request(**kwargs) -> Request: sp = SamplingParams(max_tokens=8) defaults = dict( @@ -133,6 +145,21 @@ def test_resumable_generation_stop_marks_segment_boundary() -> None: assert output.is_segment_finished is True +def test_async_chunk_resumable_stop_rearms_connector_polling() -> None: + sched = _AsyncChunkStopSchedulerStub() + session = _make_request(request_id="req-async-chunk-next-segment") + session.status = RequestStatus.FINISHED_STOPPED + session.resumable = True + + finished = sched._handle_stopped_request(session) + + assert finished is False + assert sched.enqueued == [session] + assert sched.enqueued_statuses == [RequestStatus.WAITING] + assert session.status == RequestStatus.WAITING + assert sched.num_waiting_for_streaming_input == 0 + + class TestReplaceSessionWithStreamingUpdate: def test_resets_tokens_and_prompt_from_update(self) -> None: sched = _SchedulerStub() diff --git a/tests/dfx/reliability/invalid_param_test/test_invalid_minicpmo_4_5_omni.py b/tests/dfx/reliability/invalid_param_test/test_invalid_minicpmo_4_5_omni.py index c71dc2d16..2082be10d 100644 --- a/tests/dfx/reliability/invalid_param_test/test_invalid_minicpmo_4_5_omni.py +++ b/tests/dfx/reliability/invalid_param_test/test_invalid_minicpmo_4_5_omni.py @@ -22,10 +22,7 @@ _MINICPMO_SERVER = [ model="openbmb/MiniCPM-o-4_5", stage_config_path=_MINICPMO_DEPLOY, use_stage_cli=True, - server_args=[ - "--trust-remote-code", - "--no-async-chunk", - ], + server_args=["--trust-remote-code"], ), id="minicpmo_4_5", ) diff --git a/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py b/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py index 3ab2528fb..bc8f9c30b 100644 --- a/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py +++ b/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py @@ -71,6 +71,30 @@ def test_streaming_payload_can_replace_placeholder_prompt(mocker: MockerFixture) request.update_block_hashes.assert_called_once_with() +def test_streaming_payload_can_append_exact_prompt_length(mocker: MockerFixture) -> None: + request = SimpleNamespace( + _all_token_ids=[0, 0, 7, 8], + _output_token_ids=[7, 8], + prompt_token_ids=[0, 0], + num_computed_tokens=4, + num_prompt_tokens=2, + update_block_hashes=mocker.Mock(), + ) + payload = { + "ids": {"prompt": [1, 2, 3]}, + "meta": {"next_stage_prompt_len": 3}, + } + + construct_next_stage_streaming_input_prompt(payload, request) + + assert request.prompt_token_ids == [0, 0, 7, 8, 0, 0, 0] + assert request._all_token_ids == [0, 0, 7, 8, 0, 0, 0] + assert request._output_token_ids == [] + assert request.num_computed_tokens == 4 + assert request.num_prompt_tokens == 7 + request.update_block_hashes.assert_called_once_with() + + @pytest.fixture def build_adapter(monkeypatch, mocker: MockerFixture): def _build( @@ -345,6 +369,23 @@ def test_send_single_request_struct_preserves_segment_finished(build_adapter, mo assert sent_payload.meta.is_segment_finished.item() is True +def test_send_single_request_respects_processor_receiver_boundary(build_adapter, monkeypatch): + adapter, connector = build_adapter(stage_id=1) + request = _req("req-stream", RequestStatus.WAITING, external_req_id="ext-stream") + + adapter.custom_process_next_stage_input_func = lambda **kwargs: OmniPayloadStruct( + meta=MetaStruct(is_segment_finished=torch.tensor(False, dtype=torch.bool)) + ) + monkeypatch.setattr(adapter, "cleanup", lambda *a, **kw: None) + + adapter._send_single_request( + {"multimodal_output": None, "request": request, "is_finished": False, "is_segment_finished": True} + ) + + sent_payload = connector.put.call_args.kwargs["data"] + assert sent_payload.meta.is_segment_finished.item() is False + + def test_save_async_skips_stale_resumable_chunk_until_dedup_is_reset(build_adapter): adapter, _ = build_adapter(stage_id=1) request = _req("req-stream", RequestStatus.WAITING, external_req_id="ext-stream") @@ -494,7 +535,7 @@ def test_fifo_promotion(build_adapter): adapter.process_pending_chunks(waiting_queue, running_queue) assert list(adapter._active_streams) == ["req-1", "req-2"] - assert waiting_queue == reqs[2:] + assert waiting_queue == [] assert reqs[0].status == RequestStatus.WAITING_FOR_CHUNK assert reqs[1].status == RequestStatus.WAITING_FOR_CHUNK assert reqs[2].status == RequestStatus.WAITING @@ -504,6 +545,7 @@ def test_fifo_promotion(build_adapter): # (commit c4d95fd9 — otherwise the terminal chunk deadlocks at c=8 K=2). # Simulate it here so promotion can pick up the freed slot. adapter._evict_finished_active_streams({"req-1"}) + adapter.restore_queues(waiting_queue, running_queue) adapter.process_pending_chunks(waiting_queue, running_queue) assert list(adapter._active_streams) == ["req-2", "req-3"] @@ -512,6 +554,22 @@ def test_fifo_promotion(build_adapter): assert reqs[2].status == RequestStatus.WAITING_FOR_CHUNK +def test_non_active_waiting_request_is_held_off_scheduler(build_adapter): + adapter, _ = build_adapter(stage_id=2, max_num_seqs=2, active_stream_window=1) + active = _req("req-active", RequestStatus.WAITING) + non_active = _req("req-non-active", RequestStatus.WAITING) + waiting_queue = DummyWaitingQueue([active, non_active]) + running_queue = [] + + adapter.process_pending_chunks(waiting_queue, running_queue) + + assert waiting_queue == [] + assert active.status == RequestStatus.WAITING_FOR_CHUNK + assert non_active.status == RequestStatus.WAITING + assert list(adapter._pending_load_reqs) == [active] + assert list(adapter.waiting_for_chunk_waiting_requests) == [active, non_active] + + def test_legacy_k0(build_adapter): adapter, _ = build_adapter(stage_id=1, max_num_seqs=1, active_stream_window=0) waiting_req = _req("waiting", RequestStatus.WAITING) @@ -540,12 +598,13 @@ def test_finished_releases_slot(build_adapter): adapter.process_pending_chunks(waiting_queue, running_queue) assert list(adapter._active_streams) == ["req-1"] - assert waiting_queue == [req_2] + assert waiting_queue == [] adapter.finished_requests.add("req-1") # Eviction is deferred to postprocess_scheduler_output in the runtime path # (commit c4d95fd9). Simulate it so promotion can pick up the freed slot. adapter._evict_finished_active_streams({"req-1"}) + adapter.restore_queues(waiting_queue, running_queue) adapter.process_pending_chunks(waiting_queue, running_queue) assert list(adapter._active_streams) == ["req-2"] diff --git a/tests/e2e/offline_inference/test_minicpmo_4_5.py b/tests/e2e/offline_inference/test_minicpmo_4_5.py index fe7a2d2be..a8855e04b 100644 --- a/tests/e2e/offline_inference/test_minicpmo_4_5.py +++ b/tests/e2e/offline_inference/test_minicpmo_4_5.py @@ -83,7 +83,7 @@ def test_video_to_text(omni_runner, omni_runner_handler) -> None: @hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) def test_text_to_audio(omni_runner, omni_runner_handler) -> None: - """Test processing text, generating audio output through the talker token2wav path.""" + """Test processing text and generating audio through Talker and Code2Wav.""" request_config = {"prompts": get_question("text"), "modalities": ["audio"]} omni_runner_handler.send_omni_request(request_config) diff --git a/tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py b/tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py index 8b521c378..b6d562303 100644 --- a/tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py +++ b/tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py @@ -1174,10 +1174,13 @@ async def _send_listen_only_overlap_pair( before_second_listen = state.model_listen_count overlap_started_while_active = not state.response_done(first_response_id) model_unit_ready_while_active = False + second_unit_event_index: int | None = None def record_model_unit_ready() -> None: - nonlocal model_unit_ready_while_active + nonlocal model_unit_ready_while_active, second_unit_event_index model_unit_ready_while_active = not state.response_done(first_response_id) + if model_unit_ready_while_active: + second_unit_event_index = len(state.events) await _send_pcm16( ws, @@ -1239,14 +1242,13 @@ async def _send_listen_only_overlap_pair( label=transcripts[1], ) - second_speech_index = _nth_event_index(state, "input_audio_buffer.speech_started", 2) first_done_index = _event_index_for_response(state, "response.done", first_response_id) overlap_ok = ( overlap_started_while_active and model_unit_ready_while_active - and second_speech_index is not None + and second_unit_event_index is not None and first_done_index is not None - and second_speech_index < first_done_index + and second_unit_event_index < first_done_index ) return [first_response_id, second_response_id], ["speak", second_outcome], overlap_ok @@ -1380,6 +1382,17 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]: await ws.send(json.dumps({"type": "session.close"})) await _wait_for(state, lambda: state.count("session.closed") > 0, timeout_s=20, label="session.closed") finally: + if state.count("session.created") > 0 and state.count("session.closed") == 0: + try: + await ws.send(json.dumps({"type": "session.close"})) + await _wait_for( + state, + lambda: state.count("session.closed") > 0, + timeout_s=min(args.timeout_s, 5), + label="session.closed during cleanup", + ) + except (ConnectionClosed, TimeoutError): + pass stop.set() reader.cancel() try: diff --git a/tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py b/tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py index 61a466913..e317c412c 100644 --- a/tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py +++ b/tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py @@ -19,7 +19,11 @@ SCRIPT_DIR = Path(__file__).resolve().parent if str(SCRIPT_DIR) not in sys.path: sys.path.insert(0, str(SCRIPT_DIR)) -from minicpmo_realtime_duplex_scenarios import _url_with_model, run_demo # noqa: E402 +from minicpmo_realtime_duplex_scenarios import ( # noqa: E402 + _ref_audio_data_url, + _url_with_model, + run_demo, +) def _with_resume_mode(url: str) -> str: @@ -114,7 +118,12 @@ async def _open_admission_session( args: argparse.Namespace, session_id: str, ) -> tuple[object, dict[str, object]]: - url = _url_with_model(args.url, args.model, session_id=session_id) + url = _url_with_model( + args.url, + args.model, + autostart=False if getattr(args, "ref_audio", None) else None, + session_id=session_id, + ) ws = await websockets.connect(url, max_size=64 * 1024 * 1024) await ws.send( json.dumps( @@ -125,6 +134,7 @@ async def _open_admission_session( "model": args.model, "modalities": ["audio", "text"], "extra_body": {"minicpmo45_native_duplex": True}, + **({"ref_audio": _ref_audio_data_url(args.ref_audio)} if getattr(args, "ref_audio", None) else {}), }, } ) @@ -151,7 +161,12 @@ async def _admission_probe(args: argparse.Namespace, *, limit: int) -> dict[str, accepted.append(await _open_admission_session(args, f"{prefix}-accepted-{index}")) overflow_id = f"{prefix}-overflow" - overflow_url = _url_with_model(args.url, args.model, session_id=overflow_id) + overflow_url = _url_with_model( + args.url, + args.model, + autostart=False if getattr(args, "ref_audio", None) else None, + session_id=overflow_id, + ) async with websockets.connect(overflow_url, max_size=64 * 1024 * 1024) as overflow: await overflow.send( json.dumps( @@ -162,6 +177,11 @@ async def _admission_probe(args: argparse.Namespace, *, limit: int) -> dict[str, "model": args.model, "modalities": ["audio", "text"], "extra_body": {"minicpmo45_native_duplex": True}, + **( + {"ref_audio": _ref_audio_data_url(args.ref_audio)} + if getattr(args, "ref_audio", None) + else {} + ), }, } ) @@ -207,16 +227,27 @@ async def _resume_probe( session_id: str, expect_expired: bool = False, ) -> dict[str, object]: - url = _url_with_model(args.url, args.model, session_id=session_id) + url = _url_with_model( + args.url, + args.model, + autostart=False if getattr(args, "ref_audio", None) else None, + session_id=session_id, + ) async with websockets.connect(url, max_size=64 * 1024 * 1024) as first: await first.send( json.dumps( { "type": "session.update", "session": { + "session_id": session_id, "model": args.model, "modalities": ["audio", "text"], "extra_body": {"minicpmo45_native_duplex": True}, + **( + {"ref_audio": _ref_audio_data_url(args.ref_audio)} + if getattr(args, "ref_audio", None) + else {} + ), }, } ) @@ -306,7 +337,12 @@ async def _takeover_probe( *, session_id: str, ) -> dict[str, object]: - url = _url_with_model(args.url, args.model, session_id=session_id) + url = _url_with_model( + args.url, + args.model, + autostart=False if getattr(args, "ref_audio", None) else None, + session_id=session_id, + ) first = await websockets.connect(url, max_size=64 * 1024 * 1024) second = None try: @@ -315,9 +351,15 @@ async def _takeover_probe( { "type": "session.update", "session": { + "session_id": session_id, "model": args.model, "modalities": ["audio", "text"], "extra_body": {"minicpmo45_native_duplex": True}, + **( + {"ref_audio": _ref_audio_data_url(args.ref_audio)} + if getattr(args, "ref_audio", None) + else {} + ), }, } ) @@ -451,20 +493,7 @@ async def run_multi_session(args: argparse.Namespace) -> dict[str, object]: args, session_id=f"takeover-{args.takeover_session_index}-{uuid.uuid4().hex}", ) - expiry_result = None - if args.expire_session_index is not None: - if not 0 <= args.expire_session_index < args.sessions: - raise ValueError("--expire-session-index is outside the session range") - expiry_result = await _resume_probe( - args, - session_id=f"expire-{args.expire_session_index}-{uuid.uuid4().hex}", - expect_expired=True, - ) - admission_result = ( - await _admission_probe(args, limit=args.verify_admission_limit) - if args.verify_admission_limit is not None - else None - ) + lifecycle_result = await run_lifecycle_probes(args) session_results = await asyncio.gather( *(run_demo(_demo_args(args, index)) for index in range(args.sessions)), @@ -487,16 +516,15 @@ async def run_multi_session(args: argparse.Namespace) -> dict[str, object]: and semantic_isolation_ok and (resume_result is None or resume_result.get("ok") is True) and (takeover_result is None or takeover_result.get("ok") is True) - and (expiry_result is None or expiry_result.get("ok") is True) - and (admission_result is None or admission_result.get("ok") is True) + and lifecycle_result["ok"] is True ), "session_count": args.sessions, "identity_isolation_ok": identity_isolation_ok, "semantic_isolation_ok": semantic_isolation_ok, "resume": resume_result, "takeover": takeover_result, - "expiry": expiry_result, - "admission": admission_result, + "expiry": lifecycle_result["expiry"], + "admission": lifecycle_result["admission"], "failures": failures, "sessions": completed, } @@ -507,6 +535,34 @@ async def run_multi_session(args: argparse.Namespace) -> dict[str, object]: return result +async def run_lifecycle_probes(args: argparse.Namespace) -> dict[str, object]: + """Run expiry and admission probes without requiring model output.""" + if args.sessions < 1: + raise ValueError("--sessions must be positive") + expiry_result = None + if args.expire_session_index is not None: + if not 0 <= args.expire_session_index < args.sessions: + raise ValueError("--expire-session-index is outside the session range") + expiry_result = await _resume_probe( + args, + session_id=f"expire-{args.expire_session_index}-{uuid.uuid4().hex}", + expect_expired=True, + ) + admission_result = ( + await _admission_probe(args, limit=args.verify_admission_limit) + if args.verify_admission_limit is not None + else None + ) + return { + "ok": ( + (expiry_result is None or expiry_result.get("ok") is True) + and (admission_result is None or admission_result.get("ok") is True) + ), + "expiry": expiry_result, + "admission": admission_result, + } + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--url", default="ws://127.0.0.1:8113/v1/realtime?duplex=1") diff --git a/tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py b/tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py index 3eb9405a6..57d9e433f 100644 --- a/tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py +++ b/tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py @@ -2,8 +2,8 @@ This E2E driver runs the public ``realtime_duplex_demo.py`` against a live duplex backend. Arbitrary audio defaults to the model-policy lifecycle contract. -The stronger response-required mode binds the two-response contract to a known -input checksum and expected second response. +The stronger response-required mode binds the multi-response contract to a +known input checksum and expected follow-up response. """ from __future__ import annotations @@ -39,6 +39,17 @@ def _validate_input_sha256(path: Path, expected: str) -> str: return actual +def _input_duration_s(path: Path) -> float: + with wave.open(str(path), "rb") as wav_file: + return wav_file.getnframes() / wav_file.getframerate() + + +def _client_process_timeout_s(input_wav: Path, protocol_timeout_s: float) -> float: + # The child first streams the WAV in real time, then can independently + # exhaust its post-commit and session-close protocol waits. + return _input_duration_s(input_wav) + 2 * protocol_timeout_s + 30.0 + + def _read_json(path: Path) -> dict[str, object]: if not path.is_file(): return {} @@ -211,7 +222,7 @@ def summarize_artifacts( validation_mode: str, min_responses: int, min_audio_deltas_per_response: int, - expect_second_response_substring: str | None, + expect_followup_response_substring: str | None, ) -> dict[str, object]: if validation_mode not in {"model-policy", "response-required"}: raise ValueError(f"unsupported validation mode: {validation_mode}") @@ -274,12 +285,20 @@ def summarize_artifacts( ) final_listen_after_commit = commit_index is not None and any(index > commit_index for index in listen_indices) transcript = "".join(str(summary.get("transcript") or "") for summary in response_summaries) - second_response_transcript = ( - str(response_summaries[1].get("transcript") or "") if len(response_summaries) >= 2 else "" + followup_response_transcripts = [ + str(summary.get("transcript") or "") + for summary in response_summaries[1:] + if commit_index is not None + and isinstance(summary.get("created_index"), int) + and summary["created_index"] < commit_index + ] + followup_response_transcript_ok = any( + bool(_normalize_text(transcript)) for transcript in followup_response_transcripts + ) + followup_response_transcript_expectation_ok = not expect_followup_response_substring or any( + _normalize_text(expect_followup_response_substring) in _normalize_text(transcript) + for transcript in followup_response_transcripts ) - second_response_transcript_expectation_ok = not expect_second_response_substring or _normalize_text( - expect_second_response_substring - ) in _normalize_text(second_response_transcript) error_events = [ event for event in events @@ -310,9 +329,8 @@ def summarize_artifacts( mode_contract_ok = validation_mode == "model-policy" or ( second_response_before_final_commit and listen_before_first_response - and listen_between_responses and listen_after_last_done - and second_response_transcript_expectation_ok + and followup_response_transcript_ok ) ok = common_contract_ok and mode_contract_ok return { @@ -337,9 +355,10 @@ def summarize_artifacts( "listen_after_response_before_commit": listen_after_response_before_commit, "final_listen_after_commit": final_listen_after_commit, "transcript": transcript, - "second_response_transcript": second_response_transcript, - "expect_second_response_substring": expect_second_response_substring, - "second_response_transcript_expectation_ok": second_response_transcript_expectation_ok, + "followup_response_transcripts": followup_response_transcripts, + "followup_response_transcript_ok": followup_response_transcript_ok, + "expect_followup_response_substring": expect_followup_response_substring, + "followup_response_transcript_expectation_ok": followup_response_transcript_expectation_ok, "error_count": len(error_events), "cancelled_count": cancelled_count, "compact_sequence": _compact_sequence(events), @@ -388,7 +407,7 @@ async def run_soft_interrupt(args: argparse.Namespace) -> dict[str, object]: try: stdout_bytes, stderr_bytes = await asyncio.wait_for( process.communicate(), - timeout=args.timeout_s + 30.0, + timeout=_client_process_timeout_s(input_wav, args.timeout_s), ) except TimeoutError: process.kill() @@ -406,7 +425,7 @@ async def run_soft_interrupt(args: argparse.Namespace) -> dict[str, object]: validation_mode=args.validation_mode, min_responses=args.min_responses, min_audio_deltas_per_response=args.min_audio_deltas_per_response, - expect_second_response_substring=args.expect_second_response_substring, + expect_followup_response_substring=args.expect_followup_response_substring, ) summary.update( { @@ -444,14 +463,12 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--min-responses", type=int, default=2) parser.add_argument("--min-audio-deltas-per-response", type=int, default=2) parser.add_argument("--input-sha256") - parser.add_argument("--expect-second-response-substring") + parser.add_argument("--expect-followup-response-substring") args = parser.parse_args() if args.validation_mode == "response-required" and args.min_responses < 2: parser.error("--min-responses must be at least 2") if args.validation_mode == "response-required" and not args.input_sha256: parser.error("--input-sha256 is required in response-required mode") - if args.validation_mode == "response-required" and not args.expect_second_response_substring: - parser.error("--expect-second-response-substring is required in response-required mode") if args.min_audio_deltas_per_response < 1: parser.error("--min-audio-deltas-per-response must be positive") return args diff --git a/tests/e2e/online_serving/test_minicpmo_4_5.py b/tests/e2e/online_serving/test_minicpmo_4_5.py index f875c00d8..a4fefec98 100644 --- a/tests/e2e/online_serving/test_minicpmo_4_5.py +++ b/tests/e2e/online_serving/test_minicpmo_4_5.py @@ -1,9 +1,7 @@ -""" -E2E Online tests for MiniCPM-o 4.5 model with multimodal input and audio / text output. +"""E2E online tests for MiniCPM-o 4.5 multimodal input and audio/text output. -MiniCPM-o 4.5 has ``async_chunk: false``, ``max_num_seqs: 1`` on both stages, -and the vocoder runs in-process inside the talker stage rather than as a separate -Code2Wav stage. +The batching deployment uses async chunk transfer across separate Thinker, +Talker, and Code2Wav stages. """ import os @@ -26,10 +24,7 @@ test_params = [ model=_MODEL, stage_config_path=_CI_DEPLOY, use_stage_cli=True, - server_args=[ - "--trust-remote-code", - "--no-async-chunk", - ], + server_args=["--trust-remote-code"], ), id="default", ) @@ -99,7 +94,7 @@ def test_text_to_text_001(omni_server, openai_client) -> None: def test_text_to_audio_001(omni_server, openai_client) -> None: """ Test text-only input generating text + audio output via OpenAI API. - This exercises the talker TTS region detection and in-process token2wav vocoder. + This exercises Talker TTS region detection and the Code2Wav stage. Deploy Setting: default 2GPU Input Modal: text Output Modal: text + audio @@ -133,13 +128,14 @@ def test_audio_to_text_audio_001(omni_server, openai_client) -> None: messages = dummy_messages_from_mix_data( system_prompt=get_system_prompt(), audio_data_url=audio_data_url, - content_text=get_prompt("mix"), + content_text=get_prompt(), ) request_config = { "model": omni_server.model, "messages": messages, "stream": True, + "key_words": {"text": ["Beijing"]}, } openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) diff --git a/tests/e2e/online_serving/test_minicpmo_4_5_duplex.py b/tests/e2e/online_serving/test_minicpmo_4_5_duplex.py new file mode 100644 index 000000000..2e409f689 --- /dev/null +++ b/tests/e2e/online_serving/test_minicpmo_4_5_duplex.py @@ -0,0 +1,147 @@ +"""CI coverage for the MiniCPM-o 4.5 native-duplex Realtime API.""" + +from __future__ import annotations + +import asyncio +import json +import uuid +from pathlib import Path + +import pytest +import websockets + +from tests.e2e.online_serving.minicpmo_realtime_duplex_scenarios import ( + _ref_audio_data_url, + run_demo, +) +from tests.e2e.online_serving.run_minicpmo_realtime_duplex_multi_session import ( + run_multi_session, +) +from tests.helpers.mark import hardware_test +from tests.helpers.minicpmo_4_5_duplex import ( + CORE_SERVER_PARAMS, + SERVER_PARAMS, + demo_args, + multi_session_args, + realtime_url, + resolve_ref_audio, + validated_input_wav, +) +from vllm_omni.experimental.fullduplex.client import build_realtime_url + +pytestmark = pytest.mark.omni + + +async def _receive_protocol_events(ws, required_types: set[str], *, timeout_s: float) -> list[dict[str, object]]: + async def receive() -> list[dict[str, object]]: + events: list[dict[str, object]] = [] + seen: set[str] = set() + while not required_types.issubset(seen): + raw = await ws.recv() + if not isinstance(raw, str): + continue + event = json.loads(raw) + if not isinstance(event, dict): + continue + events.append(event) + event_type = event.get("type") + if event_type == "error": + raise AssertionError(f"WebSocket protocol smoke received an error: {event}") + if isinstance(event_type, str): + seen.add(event_type) + return events + + return await asyncio.wait_for(receive(), timeout=timeout_s) + + +async def _run_protocol_smoke(*, url: str, model: str, ref_audio: Path) -> list[dict[str, object]]: + session_id = f"duplex-ci-protocol-{uuid.uuid4().hex}" + websocket_url = build_realtime_url(url, model, autostart=False, session_id=session_id) + async with websockets.connect(websocket_url, max_size=64 * 1024 * 1024) as ws: + await ws.send( + json.dumps( + { + "type": "session.update", + "session": { + "session_id": session_id, + "model": model, + "modalities": ["audio", "text"], + "ref_audio": _ref_audio_data_url(str(ref_audio)), + "extra_body": {"minicpmo45_native_duplex": True}, + }, + } + ) + ) + events = await _receive_protocol_events( + ws, + {"session.created", "session.updated"}, + timeout_s=60, + ) + await ws.send(json.dumps({"type": "session.close"})) + events.extend(await _receive_protocol_events(ws, {"session.closed"}, timeout_s=60)) + return events + + +@pytest.mark.core_model +@hardware_test(res={"cuda": "H100"}, num_cards=2) +@pytest.mark.parametrize("omni_server", CORE_SERVER_PARAMS, indirect=True) +def test_duplex_websocket_protocol_smoke(omni_server, model_prefix: str) -> None: + ref_audio = resolve_ref_audio(model_prefix) + events = asyncio.run( + _run_protocol_smoke( + url=realtime_url(omni_server), + model=omni_server.model, + ref_audio=ref_audio, + ) + ) + event_types = [event.get("type") for event in events] + assert "session.created" in event_types + assert "session.updated" in event_types + assert event_types[-1] == "session.closed" + + +@pytest.mark.advanced_model +@hardware_test(res={"cuda": "H100"}, num_cards=2) +@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) +def test_duplex_single_session_response_required(omni_server, model_prefix: str, tmp_path: Path) -> None: + result = asyncio.run( + run_demo( + demo_args( + omni_server=omni_server, + input_wav=validated_input_wav(), + ref_audio=resolve_ref_audio(model_prefix), + output_dir=tmp_path / "single_session", + ) + ) + ) + assert result["ok"] is True + assert result["audio_delta_count"] > 0 + assert result["done_count"] == 1 + assert result["error_count"] == 0 + assert result["all_audio_responses_have_transcript"] is True + assert result["transcript_delta_done_ok"] is True + + +@pytest.mark.advanced_model +@hardware_test(res={"cuda": "H100"}, num_cards=2) +@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) +def test_duplex_two_sessions_resume_and_takeover(omni_server, model_prefix: str, tmp_path: Path) -> None: + result = asyncio.run( + run_multi_session( + multi_session_args( + omni_server=omni_server, + input_wav=validated_input_wav(), + ref_audio=resolve_ref_audio(model_prefix), + output_dir=tmp_path / "multi_session", + response_required=True, + ) + ) + ) + assert result["ok"] is True + assert result["session_count"] == 2 + assert result["resume"]["ok"] is True + assert result["takeover"]["ok"] is True + assert not result["failures"] + assert all(session["audio_delta_count"] > 0 for session in result["sessions"]) + assert all(session["done_count"] == 1 for session in result["sessions"]) + assert all(session["error_count"] == 0 for session in result["sessions"]) diff --git a/tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py b/tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py new file mode 100644 index 000000000..69c266456 --- /dev/null +++ b/tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py @@ -0,0 +1,86 @@ +"""Nightly lifecycle coverage for the MiniCPM-o 4.5 native-duplex API.""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from tests.e2e.online_serving.run_minicpmo_realtime_duplex_multi_session import ( + run_lifecycle_probes, +) +from tests.e2e.online_serving.run_minicpmo_realtime_duplex_soft_interrupt import ( + run_soft_interrupt, +) +from tests.helpers.mark import hardware_test +from tests.helpers.minicpmo_4_5_duplex import ( + SERVER_PARAMS, + SOFT_INTERRUPT_SHA256, + multi_session_args, + realtime_url, + resolve_ref_audio, + validated_input_wav, + validated_soft_interrupt_wav, +) + +pytestmark = [pytest.mark.full_model, pytest.mark.omni] + + +@hardware_test(res={"cuda": "H100"}, num_cards=2) +@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) +def test_duplex_admission_and_expiry_reaper(omni_server, model_prefix: str, tmp_path: Path) -> None: + args = multi_session_args( + omni_server=omni_server, + input_wav=validated_input_wav(), + ref_audio=resolve_ref_audio(model_prefix), + output_dir=tmp_path / "admission_expiry", + response_required=False, + ) + args.sessions = 1 + args.disconnect_session_index = None + args.takeover_session_index = None + args.expire_session_index = 0 + args.verify_admission_limit = 2 + result = asyncio.run(run_lifecycle_probes(args)) + + assert result["ok"] is True, json.dumps(result, ensure_ascii=False, indent=2) + assert result["expiry"]["ok"] is True + assert result["expiry"]["error_code"] == "session_resume_expired" + assert result["admission"]["ok"] is True + assert result["admission"]["overflow_error_code"] == "resource_exhausted" + + +@hardware_test(res={"cuda": "H100"}, num_cards=2) +@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) +def test_duplex_soft_interrupt(omni_server, model_prefix: str, tmp_path: Path) -> None: + input_wav = validated_soft_interrupt_wav() + result = asyncio.run( + run_soft_interrupt( + SimpleNamespace( + url=realtime_url(omni_server), + model=omni_server.model, + input_wav=str(input_wav), + ref_audio=str(resolve_ref_audio(model_prefix)), + output_dir=str(tmp_path / "soft_interrupt"), + summary_output=None, + chunk_ms=200, + timeout_s=180.0, + require_audio=True, + no_realtime_pacing=False, + validation_mode="response-required", + min_responses=2, + min_audio_deltas_per_response=2, + input_sha256=SOFT_INTERRUPT_SHA256, + expect_followup_response_substring=None, + ) + ) + ) + + assert result["ok"] is True, json.dumps(result, ensure_ascii=False, indent=2) + assert result["error_count"] == 0 + assert result["response_lifecycle_ok"] is True + assert result["response_audio_contract_ok"] is True + assert result["followup_response_transcript_ok"] is True diff --git a/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py b/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py index 8db6cd92b..068502903 100644 --- a/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py +++ b/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py @@ -1,7 +1,7 @@ -""" -E2E Online expansion tests for MiniCPM-o 4.5 covering modality combinations, -in-process token2wav vocoder behavior, max_num_seqs=1 serial execution, -and edge-case validation. +"""E2E online expansion tests for MiniCPM-o 4.5. + +These cover modality combinations, separate Code2Wav behavior, sequential +request isolation, longer audio output, and Chinese speech. """ import os @@ -94,10 +94,8 @@ def test_text_video_to_text_001(omni_server, openai_client) -> None: @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_sequential_requests_independent(omni_server, openai_client) -> None: """ - Verify that sequential requests produce independent results and state - does not leak between requests. MiniCPM-o 4.5 has max_num_seqs=1 so - requests are processed one-at-a-time; the second request must not - receive the first request's audio or token state. + Verify that sequential requests produce independent results and that the + second request does not receive the first request's audio or token state. Deploy Setting: default 2GPU Input Modal: text (two different prompts) Output Modal: text + audio (both) @@ -138,7 +136,7 @@ def test_sequential_requests_independent(omni_server, openai_client) -> None: def test_text_to_audio_long_output_001(omni_server, openai_client) -> None: """ Test text input generating a longer audio output to exercise the - token2wav decoder across multiple frames. + Code2Wav stage across multiple frames. Deploy Setting: default 2GPU Input Modal: text (longer prompt) Output Modal: text + audio @@ -146,14 +144,14 @@ def test_text_to_audio_long_output_001(omni_server, openai_client) -> None: """ messages = dummy_messages_from_mix_data( system_prompt=get_system_prompt(), - content_text="Tell me a short story about a cat in about 50 words.", + content_text="What is the capital of China? Answer in 40 words.", ) request_config = { "model": omni_server.model, "messages": messages, "stream": True, - "key_words": {"audio": ["cat"]}, + "key_words": {"audio": ["Beijing"]}, } openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) diff --git a/tests/examples/test_minicpmo_realtime_web.py b/tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py similarity index 96% rename from tests/examples/test_minicpmo_realtime_web.py rename to tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py index 03fbc6d6d..f21035f44 100644 --- a/tests/examples/test_minicpmo_realtime_web.py +++ b/tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py @@ -12,14 +12,11 @@ import pytest pytestmark = [pytest.mark.core_model, pytest.mark.cpu] -DEMO_PATH = Path(__file__).resolve().parents[1] / "e2e/online_serving/minicpmo_realtime_duplex_scenarios.py" -MULTI_DEMO_PATH = ( - Path(__file__).resolve().parents[1] / "e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py" -) -PAIR_DEMO_PATH = Path(__file__).resolve().parents[1] / "e2e/online_serving/run_minicpmo_realtime_duplex_demo_pair.py" -SOFT_INTERRUPT_DEMO_PATH = ( - Path(__file__).resolve().parents[1] / "e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py" -) +DRIVER_DIR = Path(__file__).resolve().parent +DEMO_PATH = DRIVER_DIR / "minicpmo_realtime_duplex_scenarios.py" +MULTI_DEMO_PATH = DRIVER_DIR / "run_minicpmo_realtime_duplex_multi_session.py" +PAIR_DEMO_PATH = DRIVER_DIR / "run_minicpmo_realtime_duplex_demo_pair.py" +SOFT_INTERRUPT_DEMO_PATH = DRIVER_DIR / "run_minicpmo_realtime_duplex_soft_interrupt.py" def _load_demo_module(): @@ -338,14 +335,7 @@ def test_realtime_duplex_soft_interrupt_accepts_explicit_ref_audio(monkeypatch): assert args.validation_mode == "model-policy" -@pytest.mark.parametrize( - "extra_args", - [ - ["--expect-second-response-substring", "一加一等于二"], - ["--input-sha256", "a" * 64], - ], -) -def test_realtime_duplex_soft_interrupt_response_required_needs_bound_fixture(monkeypatch, extra_args): +def test_realtime_duplex_soft_interrupt_response_required_needs_bound_fixture(monkeypatch): demo = _load_soft_interrupt_demo_module() monkeypatch.setattr( demo.sys, @@ -360,7 +350,6 @@ def test_realtime_duplex_soft_interrupt_response_required_needs_bound_fixture(mo "ref.wav", "--validation-mode", "response-required", - *extra_args, ], ) @@ -412,7 +401,6 @@ def test_realtime_duplex_soft_interrupt_accepts_multi_delta_handoff_sequence(tmp "_client_received_at_s": 2.6, }, {"type": "response.done", "response_id": first_response_id, "_client_received_at_s": 3.0}, - {"type": "response.listen", "_client_received_at_s": 3.2}, {"type": "response.created", "response": {"id": second_response_id}, "_client_received_at_s": 4.0}, {"type": "response.speak", "response_id": second_response_id, "_client_received_at_s": 4.0}, { @@ -458,10 +446,11 @@ def test_realtime_duplex_soft_interrupt_accepts_multi_delta_handoff_sequence(tmp validation_mode="response-required", min_responses=2, min_audio_deltas_per_response=2, - expect_second_response_substring="一加一等于二", + expect_followup_response_substring="一加一等于二", ) assert summary["ok"] is True + assert summary["listen_between_responses"] is False assert summary["second_response_before_final_commit"] is True assert summary["final_listen_after_commit"] is True assert summary["response_audio_contract_ok"] is True @@ -498,7 +487,7 @@ def test_realtime_duplex_soft_interrupt_model_policy_accepts_single_response(tmp validation_mode="model-policy", min_responses=2, min_audio_deltas_per_response=2, - expect_second_response_substring=None, + expect_followup_response_substring=None, ) assert summary["ok"] is True @@ -537,14 +526,14 @@ def test_realtime_duplex_soft_interrupt_response_required_rejects_single_respons validation_mode="response-required", min_responses=2, min_audio_deltas_per_response=2, - expect_second_response_substring="一加一等于二", + expect_followup_response_substring="一加一等于二", ) assert summary["ok"] is False assert summary["enough_responses"] is False -def test_realtime_duplex_soft_interrupt_checks_expected_text_in_second_response(tmp_path): +def test_realtime_duplex_soft_interrupt_reports_text_expectation_without_gating(tmp_path): demo = _load_soft_interrupt_demo_module() output = tmp_path / "second_response_text" output.mkdir() @@ -611,11 +600,32 @@ def test_realtime_duplex_soft_interrupt_checks_expected_text_in_second_response( validation_mode="response-required", min_responses=2, min_audio_deltas_per_response=2, - expect_second_response_substring="一加一等于二", + expect_followup_response_substring="一加一等于二", ) - assert summary["ok"] is False - assert summary["second_response_transcript_expectation_ok"] is False + assert summary["ok"] is True + assert summary["followup_response_transcript_ok"] is True + assert summary["followup_response_transcript_expectation_ok"] is False + + events = [ + event + for event in events + if event.get("response_id") != second_response_id or event["type"] != "response.audio_transcript.delta" + ] + (output / "events.jsonl").write_text( + "".join(demo.json.dumps(event) + "\n" for event in events), + encoding="utf-8", + ) + missing_transcript_summary = demo.summarize_artifacts( + output_dir=output, + validation_mode="response-required", + min_responses=2, + min_audio_deltas_per_response=2, + expect_followup_response_substring="一加一等于二", + ) + + assert missing_transcript_summary["ok"] is False + assert missing_transcript_summary["followup_response_transcript_ok"] is False def test_realtime_duplex_multi_session_resume_url_disables_autostart(): @@ -1523,7 +1533,7 @@ def test_realtime_duplex_demo_waits_at_each_model_unit_and_stops_after_speak(): assert len(ws.messages) == 10 -def test_realtime_duplex_demo_listen_only_overlap_sends_next_turn_before_first_done(monkeypatch): +def test_realtime_duplex_demo_listen_only_overlap_accepts_silence_unit_before_first_done(monkeypatch): demo = _load_demo_module() state = demo.DemoState() send_calls = 0 @@ -1548,7 +1558,6 @@ def test_realtime_duplex_demo_listen_only_overlap_sends_next_turn_before_first_d del args assert not state.response_done("resp-first") kwargs["on_model_unit_ready"]() - state.add({"type": "input_audio_buffer.speech_started", "turn": 2}) _add_response_transcript(state, "resp-first", transcript="第一轮完成", audio=False) state.add({"type": "response.created", "response": {"id": "resp-second"}}) _add_response_transcript(state, "resp-second", transcript="第二轮完成", audio=True) @@ -1568,7 +1577,7 @@ def test_realtime_duplex_demo_listen_only_overlap_sends_next_turn_before_first_d ws, state, b"\x00\x00", - b"\x01\x00", + b"\x00\x00", transcripts=("first", "second"), durations_ms=(None, None), chunk_ms=200, diff --git a/tests/engine/test_multimodal_accumulation.py b/tests/engine/test_multimodal_accumulation.py index ca449154c..dbde83fef 100644 --- a/tests/engine/test_multimodal_accumulation.py +++ b/tests/engine/test_multimodal_accumulation.py @@ -18,6 +18,7 @@ def test_chunk_accumulation_policy_replaces_snapshots_and_drains_delta_state(): accumulated = MultimodalPayload.from_dict( { "audio": torch.tensor([1.0]), + "meta.segment_end": torch.tensor([0]), "meta.tts_is_last_chunk": torch.tensor([0]), "meta.turn_end": torch.tensor([0]), "meta.stable_request_value": "keep", @@ -26,6 +27,7 @@ def test_chunk_accumulation_policy_replaces_snapshots_and_drains_delta_state(): incoming = MultimodalPayload.from_dict( { "audio": torch.tensor([2.0]), + "meta.segment_end": torch.tensor([1]), "meta.tts_is_last_chunk": torch.tensor([1]), "meta.turn_end": torch.tensor([1]), } @@ -41,6 +43,7 @@ def test_chunk_accumulation_policy_replaces_snapshots_and_drains_delta_state(): drain_delta_payload(merged) assert "audio" not in merged + assert "meta.segment_end" not in merged assert "meta.tts_is_last_chunk" not in merged assert "meta.turn_end" not in merged assert merged.metadata["meta.stable_request_value"] == "keep" diff --git a/tests/engine/test_orchestrator_stage_input_bridge.py b/tests/engine/test_orchestrator_stage_input_bridge.py index 0c5bb217d..cf347e49f 100644 --- a/tests/engine/test_orchestrator_stage_input_bridge.py +++ b/tests/engine/test_orchestrator_stage_input_bridge.py @@ -7,15 +7,24 @@ import asyncio import queue from types import SimpleNamespace from typing import Any -from unittest.mock import AsyncMock +from unittest.mock import ANY, AsyncMock, MagicMock import janus import pytest from vllm.outputs import CompletionOutput, RequestOutput from vllm.sampling_params import SamplingParams -from vllm_omni.engine.orchestrator import Orchestrator, OrchestratorRequestState +from vllm_omni.engine.orchestrator import ( + Orchestrator, + OrchestratorRequestState, + _OrchestratorDuplexStagePort, +) from vllm_omni.engine.stage_pool import StagePool +from vllm_omni.experimental.fullduplex.engine.contracts import ( + DuplexStageRequestContext, + DuplexStageSubmission, +) +from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -107,11 +116,50 @@ class FakePrewarmPool: async def submit_initial(self, _request_id, _req_state, request, prompt_text=None): self.submitted.append(request) + return 0 def get_bound_replica_id(self, _request_id): return 0 +def _duplex_stage_port_submission(): + stage_pools = [] + for stage_id in range(3): + pool = SimpleNamespace( + stage_client=SimpleNamespace(default_sampling_params=SamplingParams(max_tokens=1)), + stage_vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)), + submit_initial=AsyncMock(return_value=stage_id + 10), + submit_update=AsyncMock(return_value=stage_id + 20), + ) + stage_pools.append(pool) + request_states: dict[str, OrchestratorRequestState] = {} + prewarm = AsyncMock() + port = _OrchestratorDuplexStagePort( + stage_pools=stage_pools, + request_states=request_states, + running_counter=None, + cleanup_request_ids=AsyncMock(), + async_chunk=True, + prewarm_async_chunk_stages=prewarm, + ) + context = DuplexStageRequestContext( + request_id="req-duplex", + session_id="session-duplex", + fence=DuplexFence("session-duplex"), + stage_id=0, + final_stage_id=2, + config_generation=0, + sampling_params=tuple(SamplingParams(max_tokens=1) for _ in range(3)), + ) + port.ensure_request(context) + submission = DuplexStageSubmission( + context=context, + prompt={"prompt_token_ids": [1, 2]}, + already_submitted=False, + ) + return port, stage_pools, request_states, prewarm, submission + + def _request_output(request_id: str) -> RequestOutput: completion = CompletionOutput( index=0, @@ -199,11 +247,13 @@ async def test_async_prewarm_skips_outgoing_only_stage() -> None: stage2 = FakePrewarmPool("receiver") orchestrator.stage_pools = [stage0, stage1, stage2] orchestrator._emit_tx_edge = lambda **_kwargs: None + orchestrator._record_duplex_stage_submission = MagicMock() req_state = OrchestratorRequestState( request_id="req-prewarm", prompt={"prompt_token_ids": [1, 2]}, sampling_params_list=[SamplingParams(max_tokens=1) for _ in range(3)], final_stage_id=2, + duplex_identity=SimpleNamespace(), ) await orchestrator._prewarm_async_chunk_stages( @@ -216,6 +266,23 @@ async def test_async_prewarm_skips_outgoing_only_stage() -> None: assert len(stage2.submitted) == 1 assert 1 not in req_state.stage_submit_ts assert 2 in req_state.stage_submit_ts + orchestrator._record_duplex_stage_submission.assert_called_once_with( + 2, + "req-prewarm", + 0, + req_state, + ) + + +@pytest.mark.asyncio +async def test_duplex_prewarm_runs_after_first_stage0_submission() -> None: + port, stage_pools, request_states, prewarm, submission = _duplex_stage_port_submission() + + result = await port.submit(submission) + + assert result.stage_id == 0 + stage_pools[0].submit_initial.assert_awaited_once() + prewarm.assert_awaited_once_with("req-duplex", ANY, request_states["req-duplex"]) @pytest.mark.asyncio diff --git a/tests/engine/test_output_processor.py b/tests/engine/test_output_processor.py index aef78e47e..df0911aa0 100644 --- a/tests/engine/test_output_processor.py +++ b/tests/engine/test_output_processor.py @@ -14,7 +14,7 @@ from vllm.v1.engine import FinishReason from vllm.v1.engine.output_processor import OutputProcessor as VLLMOutputProcessor from vllm_omni.outputs import output_processor -from vllm_omni.outputs.output_modality import OutputModalityNames +from vllm_omni.outputs.output_modality import OutputModality, OutputModalityNames from vllm_omni.outputs.output_processor import MultimodalOutputProcessor, OmniRequestState pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -452,6 +452,127 @@ def test_no_detokenizer_make_request_output(): assert AUDIO in result.outputs[0].multimodal_output +def test_no_detokenizer_process_outputs_returns_nonterminal_audio_chunk(monkeypatch): + """Generation-stage MM-only chunks must reach the orchestrator. + + StagePool registers generation requests without a per-request collector, + so the processor return value is the only path from Stage2 to + Orchestrator._handle_processed_outputs(). + """ + state = _make_no_detok_state(RequestOutputKind.DELTA) + processor = object.__new__(MultimodalOutputProcessor) + processor.output_modality = OutputModality.AUDIO + processor.request_states = {"r": state} + upstream_result = SimpleNamespace(request_outputs=[], reqs_to_abort=[]) + monkeypatch.setattr( + VLLMOutputProcessor, + "process_outputs", + lambda *_args, **_kwargs: upstream_result, + ) + engine_output = SimpleNamespace( + request_id="r", + multimodal_output={ + "model_outputs": torch.arange(8, dtype=torch.float32), + "sr": torch.tensor(24000, dtype=torch.int32), + }, + output_type="audio", + pooling_output=None, + new_token_ids=[], + finish_reason=None, + stop_reason=None, + kv_transfer_params=None, + routed_experts=None, + num_cached_tokens=0, + ) + + result = processor.process_outputs([engine_output]) + + assert len(result.request_outputs) == 1 + assert result.request_outputs[0].request_id == "r" + assert AUDIO in result.request_outputs[0].outputs[0].multimodal_output + + +def _make_mm_only_output_processor(monkeypatch): + processor = object.__new__(MultimodalOutputProcessor) + processor.output_modality = OutputModality.AUDIO + processor.request_states = {"r": _make_no_detok_state(RequestOutputKind.DELTA)} + monkeypatch.setattr( + VLLMOutputProcessor, + "process_outputs", + lambda *_args, **_kwargs: SimpleNamespace( + request_outputs=[], + reqs_to_abort=[], + ), + ) + + def finish_request(req_state): + processor.request_states.pop(req_state.request_id, None) + + processor._finish_request = finish_request + return processor + + +def _audio_engine_output(*, is_segment_finished: bool, is_last_chunk: bool): + return SimpleNamespace( + request_id="r", + multimodal_output={ + "model_outputs": torch.arange(8, dtype=torch.float32), + "meta.tts_is_last_chunk": torch.tensor([int(is_last_chunk)]), + }, + output_type="audio", + pooling_output=None, + new_token_ids=[], + finish_reason=FinishReason.STOP, + stop_reason=None, + kv_transfer_params=None, + routed_experts=None, + num_cached_tokens=0, + is_segment_finished=is_segment_finished, + ) + + +def test_mm_only_segment_finish_retains_request_state_for_next_audio_chunk(monkeypatch): + processor = _make_mm_only_output_processor(monkeypatch) + + first = processor.process_outputs([_audio_engine_output(is_segment_finished=True, is_last_chunk=False)]) + + assert "r" in processor.request_states + assert len(first.request_outputs) == 1 + assert first.request_outputs[0].finished is False + + second = processor.process_outputs([_audio_engine_output(is_segment_finished=False, is_last_chunk=True)]) + + assert len(second.request_outputs) == 1 + assert second.request_outputs[0].finished is True + assert "r" not in processor.request_states + + +def test_mm_only_nonfinal_audio_retains_state_without_segment_marker(monkeypatch): + processor = _make_mm_only_output_processor(monkeypatch) + + first = processor.process_outputs([_audio_engine_output(is_segment_finished=False, is_last_chunk=False)]) + + assert "r" in processor.request_states + assert len(first.request_outputs) == 1 + assert first.request_outputs[0].finished is False + + second = processor.process_outputs([_audio_engine_output(is_segment_finished=False, is_last_chunk=True)]) + + assert len(second.request_outputs) == 1 + assert second.request_outputs[0].finished is True + assert "r" not in processor.request_states + + +def test_mm_only_terminal_finish_removes_request_state(monkeypatch): + processor = _make_mm_only_output_processor(monkeypatch) + + result = processor.process_outputs([_audio_engine_output(is_segment_finished=False, is_last_chunk=True)]) + + assert len(result.request_outputs) == 1 + assert result.request_outputs[0].finished is True + assert "r" not in processor.request_states + + def test_no_detokenizer_make_request_output_with_routed_experts(): """make_request_output accepts the routed_experts arg that the multimodal output channel (_process_mm_only_outputs) passes positionally, and attaches diff --git a/tests/engine/test_single_stage_mode.py b/tests/engine/test_single_stage_mode.py index 7cf4c9d75..f90a290ca 100644 --- a/tests/engine/test_single_stage_mode.py +++ b/tests/engine/test_single_stage_mode.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +import socket from contextlib import contextmanager from types import SimpleNamespace from typing import Any @@ -11,6 +12,7 @@ import pytest from pytest_mock import MockerFixture from vllm.v1.engine.utils import EngineZmqAddresses +from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig from vllm_omni.engine.async_omni_engine import AsyncOmniEngine from vllm_omni.engine.stage_engine_core_client import StageEngineCoreClientBase from vllm_omni.engine.stage_engine_startup import ( @@ -189,6 +191,34 @@ class TestOmniMasterServerAllocation: output_port = int(alloc.output_bind_address.split(":")[-1]) assert len({handshake_port, input_port, output_port}) == 3 + def test_route_ports_remain_os_reserved_until_master_releases_them(self): + server = OmniMasterServer(master_address="127.0.0.1", master_port=15005, stage_ids=[0]) + alloc = server.get_allocation(0) + ports = [ + int(alloc.handshake_bind_address.rsplit(":", 1)[-1]), + int(alloc.input_bind_address.rsplit(":", 1)[-1]), + int(alloc.output_bind_address.rsplit(":", 1)[-1]), + ] + + for port in ports: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + with pytest.raises(OSError): + probe.bind(("127.0.0.1", port)) + + server.release_route_port_reservations(0, handshake=True, data=False) + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.bind(("127.0.0.1", ports[0])) + for port in ports[1:]: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + with pytest.raises(OSError): + probe.bind(("127.0.0.1", port)) + + server.stop() + + for port in ports[1:]: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.bind(("127.0.0.1", port)) + def test_get_zmq_addresses_returns_bind_addresses(self): server = OmniMasterServer(master_address="127.0.0.1", master_port=15002, stage_ids=[0]) alloc = server.get_allocation(0) @@ -397,6 +427,30 @@ class TestSingleStageModeDetection: assert engine.single_stage_mode is False assert engine._single_stage_id_filter is None + def test_stage_configs_path_loads_duplex_runtime_config(self, mocker: MockerFixture): + duplex_session = DuplexSessionRuntimeConfig(max_sessions=2) + get_pipeline_config = mocker.patch( + "vllm_omni.engine.async_omni_engine.StageConfigFactory.get_pipeline_config", + return_value=None, + ) + load_deploy_config = mocker.patch( + "vllm_omni.engine.async_omni_engine.load_deploy_config", + return_value=SimpleNamespace(duplex_session=duplex_session), + ) + + engine = self._make_engine_no_thread( + mocker, + stage_configs_path="/fake/duplex.yaml", + ) + + get_pipeline_config.assert_called_once_with( + model="fake-model", + trust_remote_code=False, + deploy_config_path="/fake/duplex.yaml", + ) + load_deploy_config.assert_called_once_with("/fake/duplex.yaml") + assert engine.duplex_session_config is duplex_session + def test_single_stage_mode_without_stage_id_has_no_filter(self, mocker: MockerFixture): engine = self._make_engine_no_thread( mocker, diff --git a/tests/entrypoints/openai_api/test_duplex_handler.py b/tests/entrypoints/openai_api/test_duplex_handler.py index 3ed30e7e8..9f8768bf4 100644 --- a/tests/entrypoints/openai_api/test_duplex_handler.py +++ b/tests/entrypoints/openai_api/test_duplex_handler.py @@ -2520,6 +2520,52 @@ def test_duplex_auto_response_tts_scheduler_eos_fallback(): assert results[0]["abort_data_plane_request"] is True +def test_duplex_auto_response_rearms_tts_boundary_for_next_segment_in_same_turn(): + data_plane = _test_data_plane() + request_id = "duplex-sid-next-segment-same-turn-e0-stage0" + session = DuplexSession( + session_id="sid-next-segment-same-turn", + config=DuplexSessionConfig(extra_body={"auto_response": True}), + ) + session.bind_response_turn(0) + data_plane.begin_request(request_id) + + first_segment = _duplex_tts_output( + request_id=request_id, + samples=24000, + finished=False, + tts_is_last_chunk=True, + turn_id=0, + ) + first_results = _project_data_plane( + data_plane, + {"data_plane_outputs": [first_segment]}, + session=session, + ) + + assert len(first_results) == 1 + assert first_results[0]["abort_data_plane_request"] is True + + data_plane.begin_request(request_id) + second_segment = _duplex_tts_output( + request_id=request_id, + samples=48000, + finished=False, + text="hello again", + tts_is_last_chunk=True, + turn_id=0, + ) + second_results = _project_data_plane( + data_plane, + {"data_plane_outputs": [second_segment]}, + session=session, + ) + + assert len(second_results) == 1 + assert second_results[0]["audio_data"] == "wav-24000" + assert second_results[0]["abort_data_plane_request"] is True + + def test_duplex_turn_end_is_not_swallowed_by_finished_segment_fallback(): data_plane = _test_data_plane() request_id = "duplex-sid-turn-end-after-segment-e0-stage0" @@ -6202,19 +6248,27 @@ async def test_minicpmo_native_auto_response_keeps_request_bound_for_segment_con config_timeout_s=0.1, idle_timeout_s=1, ) - ws = TimedWebSocket() + + def on_send(ws: TimedWebSocket, data: dict[str, Any]) -> None: + if data.get("type") != "session.created": + return + session = handler._registry.get("sid-native-auto-continuation") + assert session is not None + session.capabilities.chunk_period_ms = 50 + ws.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm_f32_b64(800), + "format": "pcm_f32le", + "sample_rate_hz": 16000, + } + ) + ws.put({"type": "input_audio_buffer.commit", "final": True}) + + ws = TimedWebSocket(on_send=on_send) event = _native_session_create("sid-native-auto-continuation") event["session"]["extra_body"]["auto_response"] = True ws.put(event) - ws.put( - { - "type": "input_audio_buffer.append", - "audio": _pcm_f32_b64(16000), - "format": "pcm_f32le", - "sample_rate_hz": 16000, - } - ) - ws.put({"type": "input_audio_buffer.commit", "final": True}) await handler.handle_session(ws) @@ -6225,6 +6279,112 @@ async def test_minicpmo_native_auto_response_keeps_request_bound_for_segment_con assert final is False +@pytest.mark.asyncio +async def test_minicpmo_native_auto_response_real_input_preempts_unsent_silence(): + request_id = "duplex-sid-native-real-before-silence-e0-stage0" + + class SequencedAppendEngine(FakeEngineClient): + def __init__(self) -> None: + control_result = { + "operation": "append", + "session_id": "sid-native-real-before-silence", + "ok": True, + "unsupported_count": 0, + "error_count": 0, + "stage_results": [ + { + "result": { + "data_plane_append": True, + "request_id": request_id, + "response_stage_id": 1, + } + } + ], + } + terminal_segment = _duplex_tts_output( + request_id=request_id, + samples=0, + finished=True, + tts_is_last_chunk=True, + token_ids=[151645], + ) + super().__init__( + control_result=control_result, + collect_outputs=[[terminal_segment], [], []], + collect_delay_s=0.01, + ) + self.append_sequence: list[str] = [] + self.first_real_started = asyncio.Event() + self.second_real_started = asyncio.Event() + self.real_count = 0 + + async def append_duplex_input_async(self, session_id: str, **kwargs): + kwargs.pop("expected_epoch", None) + result = await super().append_duplex_input_async(session_id, **kwargs) + payload = kwargs.get("payload") + is_silence = ( + isinstance(payload, dict) + and payload.get("audio") == OmniDuplexSessionHandler._NATIVE_SILENCE_UNIT_PAYLOAD_AUDIO + ) + if is_silence: + self.append_sequence.append("silence") + else: + self.real_count += 1 + self.append_sequence.append(f"real-{self.real_count}") + if self.real_count == 1: + self.first_real_started.set() + elif self.real_count == 2: + self.second_real_started.set() + return result + + session_created = asyncio.Event() + engine = SequencedAppendEngine() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + + def on_send(ws: TimedWebSocket, data: dict[str, Any]) -> None: + if data.get("type") != "session.created": + return + session = handler._registry.get("sid-native-real-before-silence") + assert session is not None + session.capabilities.chunk_period_ms = 50 + ws.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm_f32_b64(800), + "format": "pcm_f32le", + "sample_rate_hz": 16000, + } + ) + session_created.set() + + ws = TimedWebSocket(on_send=on_send, receive_timeout_s=2.0) + event = _native_session_create("sid-native-real-before-silence") + event["session"]["extra_body"]["auto_response"] = True + ws.put(event) + handler_task = asyncio.create_task(handler.handle_session(ws)) + + await asyncio.wait_for(session_created.wait(), timeout=1) + await asyncio.wait_for(engine.first_real_started.wait(), timeout=1) + await asyncio.sleep(0.02) + ws.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm_f32_b64(800, value=0.07), + "format": "pcm_f32le", + "sample_rate_hz": 16000, + } + ) + await asyncio.wait_for(engine.second_real_started.wait(), timeout=1) + ws.put({"type": "session.close"}) + await asyncio.wait_for(handler_task, timeout=2) + + assert engine.append_sequence[:2] == ["real-1", "real-2"] + + @pytest.mark.asyncio async def test_minicpmo_native_auto_response_real_input_waits_for_submitted_silence_tail(): request_id = "duplex-sid-native-silence-tail-e0-stage0" @@ -6294,18 +6454,26 @@ async def test_minicpmo_native_auto_response_real_input_waits_for_submitted_sile config_timeout_s=0.1, idle_timeout_s=1, ) - ws = TimedWebSocket(receive_timeout_s=2.0) + + def on_send(ws: TimedWebSocket, data: dict[str, Any]) -> None: + if data.get("type") != "session.created": + return + session = handler._registry.get("sid-native-silence-tail") + assert session is not None + session.capabilities.chunk_period_ms = 50 + ws.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm_f32_b64(800, value=0.05), + "format": "pcm_f32le", + "sample_rate_hz": 16000, + } + ) + + ws = TimedWebSocket(on_send=on_send, receive_timeout_s=2.0) event = _native_session_create("sid-native-silence-tail") event["session"]["extra_body"]["auto_response"] = True ws.put(event) - ws.put( - { - "type": "input_audio_buffer.append", - "audio": _pcm_f32_b64(16000, value=0.05), - "format": "pcm_f32le", - "sample_rate_hz": 16000, - } - ) handler_task = asyncio.create_task(handler.handle_session(ws)) await asyncio.wait_for(engine.silence_append_started.wait(), timeout=1) @@ -6330,6 +6498,135 @@ async def test_minicpmo_native_auto_response_real_input_waits_for_submitted_sile assert engine.append_sequence == ["real-1", "silence-started", "silence-returned", "real-2"] +@pytest.mark.asyncio +async def test_minicpmo_native_auto_response_preserves_silence_continuations_across_append_tails(): + request_id = "duplex-sid-native-continuation-tails-e0-stage0" + + class SequencedAppendEngine(FakeEngineClient): + def __init__(self) -> None: + control_result = { + "operation": "append", + "session_id": "sid-native-continuation-tails", + "ok": True, + "unsupported_count": 0, + "error_count": 0, + "stage_results": [ + { + "result": { + "data_plane_append": True, + "request_id": request_id, + "response_stage_id": 1, + } + } + ], + } + super().__init__(control_result=control_result) + self.real_append_started = asyncio.Event() + self.release_real_append = asyncio.Event() + self.first_silence_started = asyncio.Event() + self.release_first_silence = asyncio.Event() + self.second_silence_started = asyncio.Event() + self.release_collect = asyncio.Event() + self.silence_count = 0 + + async def append_duplex_input_async(self, session_id: str, **kwargs): + kwargs.pop("expected_epoch", None) + payload = kwargs.get("payload") + assert isinstance(payload, dict) + result = await super().append_duplex_input_async(session_id, **kwargs) + if payload.get("audio") != OmniDuplexSessionHandler._NATIVE_SILENCE_UNIT_PAYLOAD_AUDIO: + self.real_append_started.set() + await self.release_real_append.wait() + return result + + self.silence_count += 1 + if self.silence_count == 1: + self.first_silence_started.set() + await self.release_first_silence.wait() + else: + self.second_silence_started.set() + return result + + async def collect_duplex_data_plane_outputs_async( + self, + request_id: str, + *, + response_stage_id: int | None = None, + timeout: float | None = None, + ) -> list[object]: + del timeout + self.collected.append((request_id, response_stage_id)) + await self.release_collect.wait() + return [] + + session_created = asyncio.Event() + + def on_send(_ws: TimedWebSocket, data: dict[str, Any]) -> None: + if data.get("type") == "session.created": + session_created.set() + + engine = SequencedAppendEngine() + handler = OmniDuplexSessionHandler( + chat_service=FakeChatService(engine), + config_timeout_s=0.1, + idle_timeout_s=1, + ) + ws = TimedWebSocket(on_send=on_send, receive_timeout_s=2.0) + event = _native_session_create("sid-native-continuation-tails") + event["session"]["extra_body"]["auto_response"] = True + ws.put(event) + ws.put( + { + "type": "input_audio_buffer.append", + "audio": _pcm_f32_b64(16000, value=0.05), + "format": "pcm_f32le", + "sample_rate_hz": 16000, + } + ) + handler_task = asyncio.create_task(handler.handle_session(ws)) + + await asyncio.wait_for(session_created.wait(), timeout=1) + await asyncio.wait_for(engine.real_append_started.wait(), timeout=1) + session = handler._registry.get("sid-native-continuation-tails") + assert session is not None + session.capabilities.chunk_period_ms = 50 + session.bind_request(request_id) + response_id = session.begin_response(turn_id=session.turn_id) + native = handler._minicpmo_session_state(session) + scheduler = native.silence_continuation_scheduler + assert scheduler is not None + + payload = handler._native_silence_unit_payload() + payload["duplex_turn_id"] = session.turn_id + kwargs = { + "request_id": request_id, + "owner_id": f"response:{response_id}", + "response_id": response_id, + "response_owned": True, + "expected_epoch": session.epoch, + "expected_incarnation": session.incarnation, + "expected_model_turn_id": None, + "send_json": ws.send_json, + } + assert await scheduler(payload, **kwargs) is True + + next_schedule = asyncio.create_task(scheduler(payload, **kwargs)) + await asyncio.sleep(0.05) + assert not next_schedule.done() + + engine.release_real_append.set() + await asyncio.wait_for(engine.first_silence_started.wait(), timeout=1) + assert not next_schedule.done() + + engine.release_first_silence.set() + assert await asyncio.wait_for(next_schedule, timeout=1) is True + await asyncio.wait_for(engine.second_silence_started.wait(), timeout=1) + ws.put({"type": "session.close"}) + await asyncio.wait_for(handler_task, timeout=2) + + assert engine.silence_count == 2 + + @pytest.mark.asyncio async def test_minicpmo_native_duplex_uses_segment_text_metadata_for_transcript_cursor(): request_id = "duplex-sid-native-repeat-text-e0-stage0" diff --git a/tests/entrypoints/test_serve.py b/tests/entrypoints/test_serve.py index bbed8fc08..cbe69302f 100644 --- a/tests/entrypoints/test_serve.py +++ b/tests/entrypoints/test_serve.py @@ -222,6 +222,7 @@ def test_run_headless_llm_registers_with_auto_assigned_replica_id(mocker: Mocker from vllm_omni.engine.stage_engine_startup import StageRegistrationResponse stage_cfg = _make_stage_cfg(0, stage_type="llm") + stage_cfg.engine_args["async_chunk"] = True parallel_config = SimpleNamespace( data_parallel_size_local=1, data_parallel_rank=0, @@ -241,7 +242,10 @@ def test_run_headless_llm_registers_with_auto_assigned_replica_id(mocker: Mocker "vllm_omni.distributed.omni_connectors.utils.initialization.resolve_omni_kv_config_for_stage", return_value=(None, None, None), ) - mocker.patch("vllm_omni.engine.stage_init_utils.get_stage_connector_spec", return_value={}) + mock_connector_spec = mocker.patch( + "vllm_omni.engine.stage_init_utils.get_stage_connector_spec", + return_value={}, + ) mocker.patch("vllm_omni.engine.stage_init_utils.build_engine_args_dict", return_value={}) mocker.patch( "vllm_omni.engine.stage_init_utils.build_vllm_config", @@ -277,6 +281,7 @@ def test_run_headless_llm_registers_with_auto_assigned_replica_id(mocker: Mocker assert kwargs["omni_stage_config"] is stage_cfg assert kwargs["replica_id"] is None assert "socket_ownership" not in kwargs + assert mock_connector_spec.call_args.kwargs["async_chunk"] is True assert mock_manager_cls.call_count == 1 mgr_kwargs = mock_manager_cls.call_args.kwargs diff --git a/tests/examples/test_minicpmo_realtime_web_static.py b/tests/examples/test_minicpmo_realtime_web_static.py index aa9f60fe2..00859dabb 100644 --- a/tests/examples/test_minicpmo_realtime_web_static.py +++ b/tests/examples/test_minicpmo_realtime_web_static.py @@ -207,3 +207,68 @@ def test_playback_worklet_waits_before_playing_and_rebuffers_after_underrun(): capture_output=True, text=True, ) + + +def test_playback_worklet_fades_terminal_drain_to_zero(): + node = shutil.which("node") + if node is None: + pytest.skip("node is required for the AudioWorklet regression test") + + script = textwrap.dedent( + """ + const fs = require('fs'); + const vm = require('vm'); + + global.sampleRate = 1000; + const messages = []; + global.AudioWorkletProcessor = class { + constructor() { + this.port = { + onmessage: null, + postMessage: (message) => messages.push(message), + }; + } + }; + let Processor = null; + global.registerProcessor = (_name, processor) => { Processor = processor; }; + vm.runInThisContext(fs.readFileSync(process.argv[1], 'utf8')); + + const processor = new Processor(); + const pcm = new Int16Array(100); + pcm.fill(16384); + processor.handleMessage({ + type: 'audio', + pcm, + responseId: 'response-terminal', + initialBufferMs: 400, + }); + processor.handleMessage({ + type: 'drain', + responseId: 'response-terminal', + }); + + const output = new Float32Array(100); + processor.process([], [[output]]); + const assert = (condition, message) => { + if (!condition) throw new Error(message); + }; + + assert(output.some((sample) => sample !== 0), 'terminal audio must still play'); + assert(output[output.length - 1] === 0, 'terminal drain must fade the final sample to zero'); + const tail = Array.from(output.slice(-5), Math.abs); + assert( + tail.every((sample, index) => index === 0 || sample <= tail[index - 1]), + 'terminal drain fade must decrease monotonically', + ); + assert( + messages.filter((message) => message.type === 'playback-drained').length === 1, + 'terminal drain must be reported exactly once after playback', + ); + """ + ) + subprocess.run( + [node, "-e", script, str(STATIC_ROOT / "playback_worklet.js")], + check=True, + capture_output=True, + text=True, + ) diff --git a/tests/fullduplex/minicpmo45/test_input.py b/tests/fullduplex/minicpmo45/test_input.py index 035ab47ef..5562fb237 100644 --- a/tests/fullduplex/minicpmo45/test_input.py +++ b/tests/fullduplex/minicpmo45/test_input.py @@ -43,6 +43,54 @@ def test_append_emits_one_model_unit_when_multiple_units_are_buffered(): assert buffer.pending_byte_count == 16_000 * 4 +def test_speech_marker_does_not_leak_across_irregular_chunk_boundaries(): + buffer = MiniCPMO45PcmAppendBuffer() + + assert buffer.append(pcm_payload(15_000), chunk_period_ms=1_000) is None + speech_unit = buffer.append(pcm_payload(2_000, speech=False), chunk_period_ms=1_000) + silence_unit = buffer.append(pcm_payload(15_000, speech=False), chunk_period_ms=1_000) + + assert speech_unit is not None + assert speech_unit["is_speech"] is True + assert silence_unit is not None + assert silence_unit["is_speech"] is False + + +def test_force_listen_marker_does_not_leak_across_irregular_chunk_boundaries(): + buffer = MiniCPMO45PcmAppendBuffer() + forced = pcm_payload(15_000, speech=False) + forced["force_listen"] = True + + assert buffer.append(forced, chunk_period_ms=1_000) is None + forced_unit = buffer.append(pcm_payload(2_000, speech=False), chunk_period_ms=1_000) + unforced_unit = buffer.append(pcm_payload(15_000, speech=False), chunk_period_ms=1_000) + + assert forced_unit is not None + assert forced_unit["force_listen"] is True + assert unforced_unit is not None + assert unforced_unit["force_listen"] is False + + +def test_pcm_append_rollback_restores_per_span_speech_markers(): + buffer = MiniCPMO45PcmAppendBuffer() + assert buffer.append(pcm_payload(15_000), chunk_period_ms=1_000) is None + reservation = buffer.prepare_append( + pcm_payload(2_000, speech=False), + operation_id="mixed-unit", + chunk_period_ms=1_000, + ) + + assert reservation is not None + reservation.rollback() + first = buffer.append(pcm_payload(15_000, speech=False), chunk_period_ms=1_000) + second = buffer.flush(chunk_period_ms=1_000) + + assert first is not None + assert first["is_speech"] is True + assert second is not None + assert second["is_speech"] is False + + def test_commit_without_speech_does_not_synthesize_terminal_audio(): buffer = MiniCPMO45PcmAppendBuffer() buffer.append(pcm_payload(8_000, speech=False), chunk_period_ms=1_000) diff --git a/tests/helpers/minicpmo_4_5_duplex.py b/tests/helpers/minicpmo_4_5_duplex.py new file mode 100644 index 000000000..dd32c0dcf --- /dev/null +++ b/tests/helpers/minicpmo_4_5_duplex.py @@ -0,0 +1,179 @@ +"""Shared fixtures and argument builders for MiniCPM-o 4.5 duplex E2E tests.""" + +from __future__ import annotations + +import hashlib +import uuid +import wave +from pathlib import Path +from types import SimpleNamespace + +import pytest +from huggingface_hub import snapshot_download + +from tests.helpers.runtime import OmniServerParams +from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config + +MODEL = "openbmb/MiniCPM-o-4_5" +DEPLOY_CONFIG = modify_stage_config( + get_deploy_config_path("minicpmo_4_5_duplex.yaml"), + updates={ + "base_config": get_deploy_config_path("minicpmo_4_5.yaml"), + "stages": { + 0: {"kv_cache_memory_bytes": 6 * 1024 * 1024 * 1024}, + 1: {"kv_cache_memory_bytes": 512 * 1024 * 1024}, + 2: {"kv_cache_memory_bytes": 256 * 1024 * 1024}, + }, + }, +) +CORE_DEPLOY_CONFIG = modify_stage_config( + DEPLOY_CONFIG, + updates={ + "stages": { + 0: {"enforce_eager": True}, + 1: {"enforce_eager": True}, + } + }, +) +ASSET_DIR = Path(__file__).resolve().parents[1] / "assets" / "minicpmo_4_5" +RESPONSE_REQUIRED_WAV = ASSET_DIR / "response_required_16k.wav" +RESPONSE_REQUIRED_SHA256 = "2e5fd4eb3ee434ce107ee3a0591fa624a33f7683c7462f45fe651c443c9af941" +SOFT_INTERRUPT_WAV = ASSET_DIR / "soft_interrupt_16k.wav" +SOFT_INTERRUPT_SHA256 = "cadae6d0ddc510310f16f8775d6379f30a5195369ccb54ff10a8e3f9a1f4a2ea" +REF_AUDIO_RELATIVE_PATH = Path("assets") / "HT_ref_audio.wav" + +SERVER_PARAMS = [ + pytest.param( + OmniServerParams( + model=MODEL, + stage_config_path=DEPLOY_CONFIG, + use_stage_cli=True, + server_args=["--trust-remote-code"], + ), + id="three-stage-single-gpu", + ) +] +CORE_SERVER_PARAMS = [ + pytest.param( + OmniServerParams( + model=MODEL, + stage_config_path=CORE_DEPLOY_CONFIG, + use_stage_cli=True, + server_args=["--trust-remote-code"], + ), + id="three-stage-single-gpu", + ) +] + + +def validated_wav(path: Path, expected_sha256: str) -> Path: + actual_sha256 = hashlib.sha256(path.read_bytes()).hexdigest() + if actual_sha256 != expected_sha256: + raise AssertionError( + f"MiniCPM-o duplex fixture SHA256 mismatch: expected {expected_sha256}, got {actual_sha256}" + ) + with wave.open(str(path), "rb") as wav_file: + actual_format = ( + wav_file.getframerate(), + wav_file.getnchannels(), + wav_file.getsampwidth(), + wav_file.getcomptype(), + ) + expected_format = (16_000, 1, 2, "NONE") + if actual_format != expected_format: + raise AssertionError(f"MiniCPM-o duplex fixture must be 16 kHz mono PCM16, got {actual_format}") + return path + + +def validated_input_wav() -> Path: + return validated_wav(RESPONSE_REQUIRED_WAV, RESPONSE_REQUIRED_SHA256) + + +def validated_soft_interrupt_wav() -> Path: + return validated_wav(SOFT_INTERRUPT_WAV, SOFT_INTERRUPT_SHA256) + + +def resolve_ref_audio(model_prefix: str) -> Path: + if model_prefix: + model_root = Path(model_prefix) / MODEL + else: + model_root = Path(snapshot_download(MODEL, local_files_only=True)) + ref_audio = model_root / REF_AUDIO_RELATIVE_PATH + if not ref_audio.is_file(): + raise FileNotFoundError(f"MiniCPM-o checkpoint ref audio is missing: {ref_audio}") + return ref_audio + + +def realtime_url(omni_server) -> str: + return f"ws://{omni_server.host}:{omni_server.port}/v1/realtime?duplex=1" + + +def demo_args( + *, + omni_server, + input_wav: Path, + ref_audio: Path, + output_dir: Path, +) -> SimpleNamespace: + return SimpleNamespace( + url=realtime_url(omni_server), + model=omni_server.model, + session_id=f"duplex-ci-single-{uuid.uuid4().hex}", + input_wav=str(input_wav), + ref_audio=str(ref_audio), + turn_input_wav=[], + output_dir=str(output_dir), + output_audio_format="pcm16", + chunk_ms=200, + realtime_input=True, + first_turn_ms=1400, + turn_duration_ms=[], + first_turn_transcript="duplex CI speech", + omit_transcript_hints=False, + validation_mode="response-required", + temperature=0.0, + scenario="sequential", + require_audio=True, + require_distinct_inputs=False, + expect_empty_turn=[], + short_ack_ms=350, + turns=1, + timeout_s=120.0, + model_policy_settle_ms=2000, + ) + + +def multi_session_args( + *, + omni_server, + input_wav: Path, + ref_audio: Path, + output_dir: Path, + response_required: bool, +) -> SimpleNamespace: + return SimpleNamespace( + url=realtime_url(omni_server), + model=omni_server.model, + sessions=2, + input_wav=str(input_wav), + ref_audio=str(ref_audio), + session_input_wav=[], + session_expected_token=[], + turn_input_wav=[], + output_dir=str(output_dir), + realtime_input=True, + chunk_ms=200, + turns=1, + first_turn_ms=1400, + turn_duration_ms=[], + response_required=response_required, + temperature=0.0 if response_required else None, + disconnect_session_index=0, + takeover_session_index=1, + resume_after_ms=100, + expire_session_index=None, + expire_after_s=40.0, + verify_admission_limit=None, + model_policy_settle_ms=2000, + timeout_s=180.0, + ) diff --git a/tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py b/tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py index 01a51bea8..c47aa8bbf 100644 --- a/tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py +++ b/tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py @@ -19,10 +19,12 @@ class _FakeEncoder(nn.Module): def __init__(self): super().__init__() self.calls: list[int] = [] + self.last_chunk_calls: list[bool] = [] def forward_chunk(self, xs, last_chunk=False, cnn_cache=None, att_cache=None): batch, length, _ = xs.shape self.calls.append(batch) + self.last_chunk_calls.append(last_chunk) old_length = 0 if att_cache is None else att_cache.shape[3] output = xs[:, : max(1, length - 1)] cnn = xs[:, :1, :].transpose(1, 2).contiguous() @@ -212,6 +214,18 @@ def test_adapter_runs_true_batch_cfg_and_splits_request_caches(): assert cache1[0, 0, 0, 0, 0].item() == 20 +def test_fade_in_out_limits_overlap_to_available_previous_audio(): + speech = torch.arange(6, dtype=torch.float32).reshape(1, -1) + previous = torch.full((1, 3), 2.0) + window = torch.hamming_window(8, periodic=False) + + actual = BatchedToken2Wav._fade_in_out(speech, previous, window) + + expected = speech.clone() + expected[..., :3] = speech[..., :3] * window[:3] + previous * window[-3:] + torch.testing.assert_close(actual, expected) + + def test_estimator_cache_stack_split_round_trip_preserves_cfg_rows(): token2wav = _FakeToken2Wav() adapter = BatchedToken2Wav(token2wav) @@ -259,6 +273,166 @@ def test_model_preserves_output_slots_and_prefers_runtime_codes(): assert token2wav.flow.encoder.calls[-1] == 2 +def test_code2wav_projects_duplex_metadata_to_final_audio_output(): + model, token2wav = _model() + segment = _info("duplex", 0, [10, 11]) + segment_text_utf8 = torch.tensor(list(b"hello"), dtype=torch.uint8) + segment["meta"].update( + { + "duplex_epoch": 3, + "duplex_turn_id": 7, + "llm_output_text_utf8": segment_text_utf8, + "tts_is_last_chunk": True, + "turn_end": False, + } + ) + + segment_output = _forward(model, [segment]) + + assert segment_output.multimodal_outputs["meta.turn_end"][0].item() is False + # A Talker unit boundary only drains pending codec tokens. The official + # streaming path keeps Token2wav open until the assistant turn ends. + assert token2wav.flow.encoder.last_chunk_calls[-1] is False + assert "duplex" in model._states + + final = _info("duplex", 1, [12, 13], last_chunk=True) + final["meta"].update(segment["meta"]) + final["meta"]["chunk_seq"] = 1 + final["meta"]["last_chunk"] = True + final["meta"]["turn_end"] = True + output = _forward(model, [final]) + + payload = output.multimodal_outputs + assert "meta" not in payload + assert payload["meta.duplex_epoch"][0].item() == 3 + assert payload["meta.duplex_turn_id"][0].item() == 7 + torch.testing.assert_close( + payload["meta.llm_output_text_utf8"][0], + segment_text_utf8, + ) + assert payload["meta.tts_is_last_chunk"][0].item() is True + assert payload["meta.turn_end"][0].item() is True + assert token2wav.flow.encoder.last_chunk_calls[-1] is True + assert "duplex" not in model._states + + +def test_initial_empty_segment_marker_initializes_stream_without_audio(): + model, token2wav = _model() + boundary = _info("duplex", 0, []) + boundary["meta"].update( + { + "code_flat_numel": 0, + "tts_is_last_chunk": True, + "turn_end": False, + } + ) + + output = _forward(model, [boundary]) + + assert output.multimodal_outputs["model_outputs"][0].numel() == 0 + assert "duplex" in model._states + assert token2wav.hift.calls == [] + + resumed = _info( + "duplex", + 1, + [4218, 4218, 4218, 10, 11, 12, 13, 14], + ) + output = _forward(model, [resumed]) + + assert output.multimodal_outputs["model_outputs"][0].numel() > 0 + assert "duplex" in model._states + + +def test_shared_runtime_prompt_recreates_missing_file_before_second_owner(tmp_path, monkeypatch): + monkeypatch.setattr("tempfile.gettempdir", lambda: str(tmp_path)) + model, _ = _model() + reference = torch.tensor([0.0, 0.25, -0.25, 0.0]) + + first = _info("voice-a", 0, [10, 11]) + first["codes"]["ref"] = reference + first["meta"]["ref_audio_sr"] = 16000 + first["meta"].pop("prompt_cache_id") + _forward(model, [first], request_ids=["internal-a"]) + + prompt_key = model._request_prompt_keys["internal-a"] + prompt_path = Path(model._runtime_prompts[prompt_key].path) + prompt_path.unlink() + + second = _info("voice-b", 0, [12, 13]) + second["codes"]["ref"] = reference + second["meta"]["ref_audio_sr"] = 16000 + second["meta"].pop("prompt_cache_id") + _forward(model, [second], request_ids=["internal-b"]) + + assert prompt_path.is_file() + assert model._runtime_prompts[prompt_key].owners == {"internal-a", "internal-b"} + + model.on_requests_finished(["internal-a"]) + assert prompt_path.is_file() + assert model._runtime_prompts[prompt_key].owners == {"internal-b"} + + model.on_requests_finished(["internal-b"]) + assert not prompt_path.exists() + assert prompt_key not in model._runtime_prompts + + +def test_runtime_prompt_write_failure_does_not_publish_partial_file(tmp_path, monkeypatch): + monkeypatch.setattr("tempfile.gettempdir", lambda: str(tmp_path)) + model, _ = _model() + reference = torch.tensor([0.0, 0.25, -0.25, 0.0]) + + def fail_after_partial_write(path, *_args, **_kwargs): + Path(path).write_bytes(b"partial") + raise OSError("simulated write failure") + + monkeypatch.setattr( + "vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_code2wav.sf.write", + fail_after_partial_write, + ) + + with pytest.raises(OSError, match="simulated write failure"): + model._materialize_runtime_prompt(reference, 16000) + + assert len(model._runtime_prompts) == 1 + entry = next(iter(model._runtime_prompts.values())) + assert not Path(entry.path).exists() + assert list(Path(entry.path).parent.iterdir()) == [] + + +def test_runtime_prompt_files_are_isolated_between_model_instances(tmp_path, monkeypatch): + monkeypatch.setattr("tempfile.gettempdir", lambda: str(tmp_path)) + first_model, _ = _model() + second_model, _ = _model() + reference = torch.tensor([0.0, 0.25, -0.25, 0.0]) + + def runtime_ref_info(request_id: str): + info = _info(request_id, 0, [10, 11]) + info["codes"]["ref"] = reference + info["meta"]["ref_audio_sr"] = 16000 + info["meta"].pop("prompt_cache_id") + return info + + _forward(first_model, [runtime_ref_info("voice-a")], request_ids=["internal-a"]) + _forward(second_model, [runtime_ref_info("voice-b")], request_ids=["internal-b"]) + + first_key = first_model._request_prompt_keys["internal-a"] + second_key = second_model._request_prompt_keys["internal-b"] + first_path = Path(first_model._runtime_prompts[first_key].path) + second_path = Path(second_model._runtime_prompts[second_key].path) + assert first_key == second_key + assert first_path != second_path + assert first_path.is_file() + assert second_path.is_file() + + first_model.on_requests_finished(["internal-a"]) + assert not first_path.exists() + assert second_path.is_file() + + second_model.on_requests_finished(["internal-b"]) + assert not second_path.exists() + + def test_mixed_final_exact_buckets_keep_order_and_release_only_final_states(): model, _ = _model() _forward( @@ -485,27 +659,37 @@ def test_reference_voice_and_duplex_metadata_follow_request_lifecycle(): model, _ = _model() first = _info("voice-a", 0, [1, 2]) first["codes"]["ref"] = torch.linspace(-0.1, 0.1, 160) + segment_text_utf8 = torch.tensor(list(b"hello"), dtype=torch.uint8) first["meta"].update( ref_audio_sr=16000, - native_duplex_segment_text="hello", + llm_output_text_utf8=segment_text_utf8, duplex_turn_id=7, duplex_epoch=3, ) first["meta"].pop("prompt_cache_id") output = _forward(model, [first]) - prompt_cache_id, prompt_wav = model._owned_prompt_wavs["voice-a"] - assert prompt_cache_id.startswith("voice-a:") + prompt_key = model._request_prompt_keys["voice-a"] + prompt = model._runtime_prompts[prompt_key] + prompt_cache_id, prompt_wav = prompt.cache_id, prompt.path + assert prompt_cache_id.startswith("runtime-ref-") assert Path(prompt_wav).is_file() - assert bytes(output.multimodal_outputs["meta.llm_output_text_utf8"][0].tolist()).decode() == "hello" + torch.testing.assert_close( + output.multimodal_outputs["meta.llm_output_text_utf8"][0], + segment_text_utf8, + ) assert output.multimodal_outputs["meta.duplex_turn_id"][0].item() == 7 assert output.multimodal_outputs["meta.duplex_epoch"][0].item() == 3 final = _info("voice-a", 1, [3, 4], last_chunk=True) final["meta"].pop("prompt_cache_id") + final["meta"]["tts_is_last_chunk"] = True output = _forward(model, [final]) assert output.multimodal_outputs["meta.tts_is_last_chunk"][0].item() is True - assert "voice-a" not in model._owned_prompt_wavs + assert model._request_prompt_keys["voice-a"] == prompt_key + model.on_requests_finished(["voice-a"]) + assert "voice-a" not in model._request_prompt_keys + assert prompt_key not in model._runtime_prompts assert not Path(prompt_wav).exists() assert (prompt_cache_id, prompt_wav) not in model.backend._prompt_features diff --git a/tests/model_executor/models/minicpmo_4_5/test_talker_batching.py b/tests/model_executor/models/minicpmo_4_5/test_talker_batching.py index 2e10041d5..975f2cdaa 100644 --- a/tests/model_executor/models/minicpmo_4_5/test_talker_batching.py +++ b/tests/model_executor/models/minicpmo_4_5/test_talker_batching.py @@ -14,6 +14,7 @@ from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import ( from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( MiniCPMO45OmniTTSForConditionalGeneration, _max_audio_tokens, + _restore_weight_norm_weight, ) from vllm_omni.utils.mm_outputs import to_payload_element @@ -56,6 +57,8 @@ def _make_talker() -> MiniCPMO45OmniTTSForConditionalGeneration: talker._request_generators = {} talker._request_audio_states = {} talker._deferred_cleanup_ids = set() + talker._codec_min_tokens = 50 + talker._codec_seed = 42 return talker @@ -81,6 +84,23 @@ def test_audio_token_limit_scales_with_condition_length( assert _max_audio_tokens(condition_tokens) == expected +def test_weight_norm_restore_matches_checkpoint_parametrization_in_bfloat16() -> None: + generator = torch.Generator().manual_seed(42) + weight_v = torch.randn(8, 16, generator=generator, dtype=torch.bfloat16) + weight_g = torch.rand(8, 1, generator=generator, dtype=torch.bfloat16) + linear = nn.utils.parametrizations.weight_norm( + nn.Linear(16, 8, bias=False, dtype=torch.bfloat16), + dim=0, + ) + with torch.no_grad(): + linear.parametrizations.weight.original0.copy_(weight_g) + linear.parametrizations.weight.original1.copy_(weight_v) + + restored = _restore_weight_norm_weight(weight_g, weight_v) + + assert torch.equal(restored, linear.weight) + + def test_talker_emits_request_aligned_codec_deltas_after_compaction(mocker) -> None: talker = _make_talker() seen: list[tuple[str, list[float], list[int]]] = [] @@ -114,9 +134,73 @@ def test_talker_emits_request_aligned_codec_deltas_after_compaction(mocker) -> N assert _routed(output, 0)["codes"]["audio"].tolist() == [[2]] assert _routed(output, 1)["codes"]["audio"].tolist() == [[3]] assert _routed(output, 0)["meta"]["finished"].item() is False + assert set(output.multimodal_outputs["meta"]) == {"finished"} assert talker.compute_logits(output.text_hidden_states).argmax(dim=-1).tolist() == [0, 0] +def test_talker_projects_request_aligned_duplex_metadata(mocker) -> None: + talker = _make_talker() + mocker.patch.object(talker, "_sample_audio_code", return_value=torch.tensor(2)) + infos = [ + { + "request_id": "req-a", + "native_duplex": True, + "duplex": {"epoch": 3, "turn_id": 7}, + "ids": {"tts": [41]}, + "meta": { + "native_duplex_segment_text": "first", + "turn_eos_token_id": 99, + }, + "audio_codes": {"accumulated": torch.empty(0, dtype=torch.long)}, + }, + { + "request_id": "req-b", + "native_duplex": True, + "duplex": {"epoch": 4, "turn_id": 8}, + "ids": {"tts": [42, 99]}, + "meta": { + "native_duplex_segment_text": "second", + "turn_eos_token_id": 99, + }, + "audio_codes": {"accumulated": torch.empty(0, dtype=torch.long)}, + }, + ] + + output = talker.make_omni_output( + torch.ones(2, 2), + model_intermediate_buffer=infos, + request_token_spans=[(0, 1), (1, 2)], + ) + + meta = output.multimodal_outputs["meta"] + assert [value.item() for value in meta["native_duplex"]] == [True, True] + assert [value.item() for value in meta["duplex_epoch"]] == [3, 4] + assert [value.item() for value in meta["duplex_turn_id"]] == [7, 8] + assert "native_duplex_segment_text" not in meta + assert [bytes(value.tolist()).decode("utf-8") for value in meta["llm_output_text_utf8"]] == [ + "first", + "second", + ] + assert [value.item() for value in meta["turn_end"]] == [False, True] + + +def test_talker_rejects_native_duplex_without_fence_identity(mocker) -> None: + talker = _make_talker() + mocker.patch.object(talker, "_sample_audio_code", return_value=torch.tensor(2)) + info = { + "request_id": "req-missing-fence", + "native_duplex": True, + "audio_codes": {"accumulated": torch.empty(0, dtype=torch.long)}, + } + + with pytest.raises(RuntimeError, match="requires non-negative integer epoch and turn_id"): + talker.make_omni_output( + torch.ones(1, 2), + model_intermediate_buffer=[info], + request_token_spans=[(0, 1)], + ) + + def test_incomplete_prefill_emits_no_code_and_does_not_advance_state(mocker) -> None: talker = _make_talker() sample = mocker.patch.object(talker, "_sample_audio_code", return_value=torch.tensor(2)) @@ -179,7 +263,7 @@ def test_eos_is_terminal_once_and_never_enters_codec_history(mocker) -> None: assert talker.compute_logits(second.text_hidden_states).argmax(dim=-1).tolist() == [1] -def test_max_token_terminal_includes_only_new_codec_delta(mocker) -> None: +def test_max_token_terminal_drops_unconsumed_codec_delta(mocker) -> None: talker = _make_talker() mocker.patch.object(talker, "_sample_audio_code", return_value=torch.tensor(3)) info = { @@ -194,8 +278,10 @@ def test_max_token_terminal_includes_only_new_codec_delta(mocker) -> None: request_token_spans=[(0, 1)], ) - assert info["audio_codes"]["accumulated"].tolist() == [4, 5, 3] - assert output.multimodal_outputs["codes"]["audio"][0].tolist() == [[3]] + # MiniCPMTTS.generate_chunk samples once at the max-token boundary to + # advance RNG state, but the sampled code is not fed into KV or returned. + assert info["audio_codes"]["accumulated"].tolist() == [4, 5] + assert output.multimodal_outputs["codes"]["audio"][0].shape == (0, 1) assert output.multimodal_outputs["meta"]["finished"][0].item() is True assert talker.compute_logits(output.text_hidden_states).argmax(dim=-1).tolist() == [1] @@ -268,6 +354,94 @@ def test_empty_speech_segment_finishes_without_sampling_codes() -> None: assert decode_embeds.shape == (1, 4) +def test_chunked_prefill_tail_aligns_condition_with_prompt_length(mocker) -> None: + talker = _make_talker() + talker.emb_text = nn.Embedding(1, 2) + condition = torch.arange(18, dtype=torch.float32).reshape(9, 2) + mocker.patch.object(talker, "_build_condition_embeddings", return_value=condition) + + _, embeds, _ = talker.preprocess( + torch.zeros(9, dtype=torch.long), + None, + _omni_is_prefill=True, + _omni_num_computed_tokens=59, + _omni_prompt_len=68, + request_id="req-chunked-prefill", + tts_token_ids=torch.tensor([1]), + tts_hidden_states=torch.ones(1, 2), + ) + + assert torch.equal(embeds, condition) + state = talker._request_audio_states["req-chunked-prefill"] + assert state["min_tokens"] == 50 + assert state["max_tokens"] == 64 + + +@pytest.mark.parametrize( + ("meta", "expected_min_tokens"), + [ + ({"turn_start": True}, 0), + ({}, 26), + ({"turn_end": True}, 0), + ], +) +def test_native_duplex_prefill_uses_official_chunk_limits( + mocker, + meta, + expected_min_tokens, +) -> None: + talker = _make_talker() + talker.emb_text = nn.Embedding(1, 2) + mocker.patch.object( + talker, + "_build_condition_embeddings", + return_value=torch.ones(3, 2), + ) + + talker.preprocess( + torch.zeros(3, dtype=torch.long), + None, + _omni_is_prefill=True, + request_id="req-duplex-chunk", + native_duplex=True, + meta=meta, + tts_token_ids=torch.tensor([1]), + tts_hidden_states=torch.ones(1, 2), + ) + + state = talker._request_audio_states["req-duplex-chunk"] + assert state["min_tokens"] == expected_min_tokens + assert state["max_tokens"] == 26 + + +def test_native_duplex_condition_matches_official_text_plus_audio_bos() -> None: + talker = _make_talker() + talker.emb_text = nn.Embedding(16, 2) + talker.projector_semantic = nn.Identity() + talker._normalize = False + talker._text_eos_id = 14 + talker._tts_bos_id = 15 + with torch.no_grad(): + talker.emb_text.weight.copy_(torch.arange(32, dtype=torch.float32).reshape(16, 2)) + + token_ids = torch.tensor([2, 3]) + hidden_states = torch.tensor([[0.5, 1.0], [1.5, 2.0]]) + + condition = talker._build_condition_embeddings( + token_ids, + hidden_states, + native_duplex=True, + ) + + expected_text = talker.emb_text(token_ids) + hidden_states + expected = torch.cat( + [expected_text, talker.emb_text(torch.tensor([talker._tts_bos_id]))], + dim=0, + ) + assert torch.equal(condition, expected) + assert condition.shape[0] == token_ids.shape[0] + 1 + + def test_request_cleanup_evicts_ar_rng_and_decode_state() -> None: talker = _make_talker() talker._request_generators["req-done"] = torch.Generator() diff --git a/tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py b/tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py index 25124ed6f..f0da872b9 100644 --- a/tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py +++ b/tests/model_executor/stage_input_processors/test_minicpmo_4_5_async_chunk.py @@ -43,6 +43,27 @@ def _delta(*codes: int): } +def _duplex_delta( + *codes: int, + epoch: int = 3, + turn_id: int = 7, + text: str = "segment", + turn_end: bool = False, +): + text_utf8 = torch.tensor(list(text.encode("utf-8")), dtype=torch.uint8) + return { + "codes": {"audio": torch.tensor(codes, dtype=torch.long).reshape(-1, 1)}, + "meta": { + "finished": torch.tensor(False), + "native_duplex": torch.tensor(True), + "duplex_epoch": torch.tensor(epoch), + "duplex_turn_id": torch.tensor(turn_id), + "llm_output_text_utf8": text_utf8, + "turn_end": torch.tensor(turn_end), + }, + } + + def _codes(payload) -> list[int]: assert payload.codes is not None assert isinstance(payload.codes.audio, torch.Tensor) @@ -108,29 +129,57 @@ def test_short_final_flushes_silence_prefix_and_tail() -> None: assert final.meta.finished.item() is True +def test_duplex_turn_end_waits_for_terminal_codec_flush() -> None: + manager = _manager() + request = _request("req-duplex") + + body = tts2code2wav_async_chunk( + manager, + _duplex_delta(*range(25), turn_end=True), + request, + False, + ) + final = tts2code2wav_async_chunk( + manager, + _duplex_delta(turn_end=True), + request, + True, + ) + + assert body is not None + assert body.meta.last_chunk is False + assert body.meta.turn_end is False + assert final is not None + assert _codes(final) == [22, 23, 24] + assert final.meta.last_chunk is True + assert final.meta.turn_end is True + + def test_first_chunk_forwards_reference_voice_and_duplex_identity() -> None: manager = _manager() request = _request("req") request.additional_information = { "codes": {"ref": [0.1, -0.1]}, - "meta": { - "ref_audio_sr": 16000, - "native_duplex_segment_text": "hello", - "segment_end": True, - "turn_end": True, - }, - "duplex": {"epoch": 3, "model_turn_id": 7}, + "meta": {"ref_audio_sr": 16000}, } - payload = tts2code2wav_async_chunk(manager, _delta(*range(7)), request, True) + payload = tts2code2wav_async_chunk( + manager, + _duplex_delta(*range(7), text="hello", turn_end=True), + request, + True, + ) assert payload is not None assert payload.codes.ref.tolist() == pytest.approx([0.1, -0.1]) assert payload.meta.ref_audio_sr == 16000 - assert payload.meta.native_duplex_segment_text == "hello" + torch.testing.assert_close( + payload.meta.llm_output_text_utf8, + torch.tensor(list(b"hello"), dtype=torch.uint8), + ) assert payload.meta.duplex_epoch == 3 assert payload.meta.duplex_turn_id == 7 - assert payload.meta.segment_end is True + assert payload.meta.tts_is_last_chunk is True assert payload.meta.turn_end is True @@ -211,6 +260,186 @@ def test_empty_final_releases_wait_gate_once() -> None: assert duplicate is None +def test_empty_duplex_boundary_uses_zero_length_transport_placeholder() -> None: + manager = _manager() + + boundary = tts2code2wav_async_chunk( + manager, + _duplex_delta(text="boundary"), + _request("req-duplex"), + True, + ) + + assert boundary is not None + assert _codes(boundary) == [0] + assert boundary.meta.code_flat_numel == 0 + assert boundary.meta.last_chunk is False + assert boundary.meta.is_segment_finished.item() is False + torch.testing.assert_close( + boundary.meta.llm_output_text_utf8, + torch.tensor(list(b"boundary"), dtype=torch.uint8), + ) + + +def test_duplex_segments_preserve_stream_state_without_closing_turn() -> None: + manager = _manager() + request = _request("req-duplex") + request.additional_information = { + "codes": {"ref": [0.1, 0.2, 0.3]}, + "meta": {"ref_audio_sr": 16000}, + } + + first = tts2code2wav_async_chunk( + manager, + _duplex_delta(10, 11, text="first"), + request, + True, + ) + second = tts2code2wav_async_chunk( + manager, + _duplex_delta(12, 13, text="second"), + request, + True, + ) + + assert first is not None + assert second is not None + assert first.meta.last_chunk is False + assert second.meta.last_chunk is False + assert first.codes is not None + assert torch.allclose(first.codes.ref, torch.tensor([0.1, 0.2, 0.3])) + assert first.meta.ref_audio_sr == 16000 + assert second.codes is not None + assert second.codes.ref is None + assert second.meta.cache_epoch == first.meta.cache_epoch + assert second.meta.chunk_seq == first.meta.chunk_seq + 1 + assert second.meta.duplex_epoch == 3 + assert second.meta.duplex_turn_id == 7 + torch.testing.assert_close( + second.meta.llm_output_text_utf8, + torch.tensor(list(b"second"), dtype=torch.uint8), + ) + assert second.meta.tts_is_last_chunk is True + assert second.meta.turn_end is False + assert first.meta.is_segment_finished.item() is False + assert second.meta.is_segment_finished.item() is False + + +def test_duplex_short_units_wait_for_minimum_stream_body() -> None: + manager = _manager() + request = _request("req-duplex") + + first = tts2code2wav_async_chunk( + manager, + _duplex_delta(10, 11, 12, text="first"), + request, + True, + ) + second = tts2code2wav_async_chunk( + manager, + _duplex_delta(13, 14, text="first"), + request, + True, + ) + + assert first is not None + assert _codes(first) == [0] + assert first.meta.code_flat_numel == 0 + assert first.meta.last_chunk is False + assert first.meta.tts_is_last_chunk is True + assert second is not None + assert _codes(second) == [4218, 4218, 4218, 10, 11, 12, 13, 14] + assert second.meta.code_flat_numel == 8 + torch.testing.assert_close( + second.meta.llm_output_text_utf8, + torch.tensor(list(b"firstfirst"), dtype=torch.uint8), + ) + + +def test_duplex_empty_finish_callback_does_not_replay_previous_text() -> None: + manager = _manager() + request = _request("req-duplex") + + first = tts2code2wav_async_chunk( + manager, + _duplex_delta(*range(25), text="first"), + request, + False, + ) + boundary = tts2code2wav_async_chunk( + manager, + _duplex_delta(text="first"), + request, + True, + ) + second = tts2code2wav_async_chunk( + manager, + _duplex_delta(*range(25, 50), text="second"), + request, + False, + ) + + assert first is not None + assert boundary is not None + assert boundary.meta.code_flat_numel == 0 + assert second is not None + torch.testing.assert_close( + second.meta.llm_output_text_utf8, + torch.tensor(list(b"second"), dtype=torch.uint8), + ) + + +def test_duplex_short_tail_does_not_replay_previous_segment_text() -> None: + manager = _manager() + request = _request("req-duplex") + + def chunk(codes, text: str, finished: bool): + return tts2code2wav_async_chunk( + manager, + _duplex_delta(*codes, text=text), + request, + finished, + ) + + first = chunk(range(25), "和上海之间", False) + first_tail = chunk([25, 26], "和上海之间", True) + assert first is not None + assert first.meta.llm_output_text_utf8.tolist() == list("和上海之间".encode()) + assert first_tail is not None + assert first_tail.meta.code_flat_numel == 0 + assert chunk([27, 28, 29], "的距离大约是", False) is None + next_flush = chunk([], "的距离大约是", True) + assert next_flush is not None + assert next_flush.meta.llm_output_text_utf8.tolist() == list("的距离大约是".encode()) + + +def test_duplex_turn_end_closes_epoch_and_next_turn_restarts_sequence() -> None: + manager = _manager() + request = _request("req-duplex") + + turn_end = tts2code2wav_async_chunk( + manager, + _duplex_delta(14, turn_id=7, turn_end=True), + request, + True, + ) + next_turn = tts2code2wav_async_chunk( + manager, + _duplex_delta(20, turn_id=8), + request, + True, + ) + + assert turn_end is not None + assert next_turn is not None + assert turn_end.meta.last_chunk is True + assert turn_end.meta.turn_end is True + assert turn_end.meta.is_segment_finished.item() is True + assert next_turn.meta.cache_epoch == turn_end.meta.cache_epoch + 1 + assert next_turn.meta.chunk_seq == 0 + assert next_turn.meta.last_chunk is False + + def test_staggered_requests_keep_accumulators_isolated() -> None: manager = _manager() req_a = _request("a") diff --git a/tests/model_executor/stage_input_processors/test_minicpmo_4_5_omni.py b/tests/model_executor/stage_input_processors/test_minicpmo_4_5_omni.py index c8381d881..652215a08 100644 --- a/tests/model_executor/stage_input_processors/test_minicpmo_4_5_omni.py +++ b/tests/model_executor/stage_input_processors/test_minicpmo_4_5_omni.py @@ -128,13 +128,125 @@ def test_native_duplex_speak_segment_reaches_split_talker() -> None: info = converted["model_intermediate_buffer"] assert info["native_duplex"] is True assert info["ids"]["tts"] == [21, 22] - assert converted["prompt_token_ids"] == [0, 0, 0, 0] + assert converted["prompt_token_ids"] == [0, 0, 0] assert info["meta"]["replace_streaming_prompt"] is True + assert info["meta"]["next_stage_prompt_len"] == 3 + assert info["meta"]["turn_start"] is True assert info["meta"]["segment_end"] is True assert info["duplex"]["epoch"] == 3 assert info["duplex"]["turn_id"] == 7 +def test_native_duplex_continuation_appends_only_new_talker_condition() -> None: + prompt_ids = [101, 102] + token_ids = { + "tts_bos_token_id": 9301, + "tts_eos_token_id": 9302, + "listen_token_id": 9303, + "speak_token_id": 9304, + "chunk_eos_token_id": 9308, + "chunk_tts_eos_token_id": 9309, + "turn_eos_token_id": 9310, + } + context = SimpleNamespace( + bridge_states={ + "duplex": { + "epoch": 3, + "model_turn_id": 7, + } + } + ) + + first_ids = [9304, 21, 22, 9308] + first = _output( + prompt_ids=prompt_ids, + output_ids=first_ids, + latent=torch.arange(24, dtype=torch.float32).reshape(6, 4), + multimodal_output={ + "duplex_prompt_token_ids": prompt_ids, + "meta": token_ids, + }, + ) + second_ids = [*first_ids, 9304, 23, 24, 9308] + second = _output( + prompt_ids=prompt_ids, + output_ids=second_ids, + latent=torch.arange(40, dtype=torch.float32).reshape(10, 4), + multimodal_output={ + "duplex_prompt_token_ids": prompt_ids, + "meta": token_ids, + }, + ) + third_ids = [*second_ids, 9304, 25, 26, 9308] + third = _output( + prompt_ids=prompt_ids, + output_ids=third_ids, + latent=torch.arange(56, dtype=torch.float32).reshape(14, 4), + multimodal_output={ + "duplex_prompt_token_ids": prompt_ids, + "meta": token_ids, + }, + ) + + first_input = llm2tts([first], prompt=[{}], _streaming_context=context)[0] + second_input = llm2tts([second], prompt=[{}], _streaming_context=context)[0] + context.bridge_states["duplex"]["model_turn_id"] = 8 + third_input = llm2tts([third], prompt=[{}], _streaming_context=context)[0] + + assert first_input["model_intermediate_buffer"]["ids"]["tts"] == [21, 22] + assert second_input["model_intermediate_buffer"]["ids"]["tts"] == [23, 24] + assert third_input["model_intermediate_buffer"]["ids"]["tts"] == [25, 26] + assert first_input["model_intermediate_buffer"]["meta"]["turn_start"] is True + assert second_input["model_intermediate_buffer"]["meta"]["turn_start"] is False + assert third_input["model_intermediate_buffer"]["meta"]["turn_start"] is True + assert first_input["model_intermediate_buffer"]["meta"]["replace_streaming_prompt"] is True + assert "replace_streaming_prompt" not in second_input["model_intermediate_buffer"]["meta"] + assert third_input["model_intermediate_buffer"]["meta"]["replace_streaming_prompt"] is True + assert second_input["model_intermediate_buffer"]["meta"]["next_stage_prompt_len"] == 3 + assert second_input["prompt_token_ids"] == [0, 0, 0] + + +def test_native_duplex_transcript_decodes_the_talker_condition_slice() -> None: + prompt_ids = [101, 102] + metadata = { + "tts_bos_token_id": 9301, + "tts_eos_token_id": 9302, + "listen_token_id": 9303, + "speak_token_id": 9304, + "chunk_eos_token_id": 9308, + "chunk_tts_eos_token_id": 9309, + "turn_eos_token_id": 9310, + } + token_text = {21: "杭州", 22: "和", 23: "州和", 24: "上海之间", 25: "大"} + + def output(token_ids: list[int]): + return _output( + prompt_ids=prompt_ids, + output_ids=token_ids, + latent=torch.zeros((len(prompt_ids) + len(token_ids), 1)), + multimodal_output={ + "duplex_prompt_token_ids": prompt_ids, + "meta": metadata, + }, + ) + + context = SimpleNamespace( + bridge_states={"duplex": {"epoch": 3, "model_turn_id": 7}}, + source_token_decoder=lambda ids, **_: "".join(token_text.get(int(token_id), "") for token_id in ids), + ) + first_ids = [9304, 21, 22, 9308] + first_info = llm2tts([output(first_ids)], prompt=[{}], _streaming_context=context)[0]["model_intermediate_buffer"] + second_info = llm2tts( + [output([*first_ids, 23, 24, 25, 9308])], + prompt=[{}], + _streaming_context=context, + )[0]["model_intermediate_buffer"] + + assert first_info["meta"]["native_duplex_segment_text"] == "杭州和" + assert second_info["ids"]["tts"] == [24, 25] + assert second_info["meta"]["native_duplex_segment_text"] == "上海之间大" + + def test_native_duplex_requires_tokenizer_boundary_metadata() -> None: latent = torch.zeros((3, 4)) source = _output( diff --git a/tests/model_executor/stage_input_processors/test_qwen3_omni_streaming_helpers.py b/tests/model_executor/stage_input_processors/test_qwen3_omni_streaming_helpers.py index 8a6dfbe94..96a938513 100644 --- a/tests/model_executor/stage_input_processors/test_qwen3_omni_streaming_helpers.py +++ b/tests/model_executor/stage_input_processors/test_qwen3_omni_streaming_helpers.py @@ -4,6 +4,7 @@ from __future__ import annotations +from collections import defaultdict from types import SimpleNamespace import pytest @@ -227,6 +228,47 @@ def test_talker2code2wav_full_payload_keeps_all_zero_codec_rows() -> None: assert "code_predictor_codes" not in payload +def test_talker2code2wav_async_chunk_flushes_cached_tail_on_stop_token() -> None: + request_id = "codec_tail" + stop_token_id = 999 + transfer_manager = SimpleNamespace( + code_prompt_token_ids=defaultdict(list), + put_req_chunk=defaultdict(int, {request_id: 2}), + connector=SimpleNamespace( + config={ + "extra": { + "initial_codec_chunk_frames": 4, + "codec_chunk_frames": 25, + "codec_left_context_frames": 25, + } + } + ), + ) + transfer_manager.code_prompt_token_ids[request_id] = [ + torch.tensor([[frame, frame + 100]], dtype=torch.long) for frame in range(50) + ] + request = SimpleNamespace( + external_req_id=request_id, + sampling_params=SimpleNamespace( + stop_token_ids=[stop_token_id], + stop_token_id=None, + ), + ) + + payload = q3.talker2code2wav_async_chunk( + transfer_manager, + {"codes": {"audio": torch.tensor([[stop_token_id, 0]], dtype=torch.long)}}, + request, + is_finished=True, + ) + + assert payload is not None + assert payload.meta.finished.item() is True + # 4 initial frames and one 25-frame chunk were already emitted. The final + # payload contains 25 frames of left context plus the remaining 21 frames. + assert payload.codes.audio.numel() == (25 + 21) * 2 + + def test_thinker2talker_full_payload_packs_complete_tensors() -> None: """Full-payload path drops the terminal thinker row before talker prefill.""" request = SimpleNamespace( diff --git a/tests/test_config_factory.py b/tests/test_config_factory.py index aa3ab604e..61d870e98 100644 --- a/tests/test_config_factory.py +++ b/tests/test_config_factory.py @@ -1160,8 +1160,10 @@ class TestDeployConfigLoading: assert [stage.to_omegaconf().session_mode for stage in stages] == ["duplex", "duplex", "duplex"] assert [stage.yaml_engine_args["async_scheduling"] for stage in stages] == [False, False, False] assert all("Async" not in (stage.scheduler_cls or "") for stage in stages) + assert [stage.devices for stage in deploy.stages] == ["0", "0", "0"] assert deploy.stages[1].enforce_eager is False assert stages[1].yaml_extras["default_sampling_params"]["max_tokens"] == 2048 + assert stages[1].yaml_extras["default_sampling_params"]["min_tokens"] == 0 assert stages[1].yaml_extras["default_sampling_params"]["stop_token_ids"] == [1] @pytest.mark.parametrize( diff --git a/tests/worker/test_native_duplex_hooks.py b/tests/worker/test_native_duplex_hooks.py index 661656b7e..383379202 100644 --- a/tests/worker/test_native_duplex_hooks.py +++ b/tests/worker/test_native_duplex_hooks.py @@ -276,16 +276,16 @@ def test_minicpmo_model_hook_same_speech_row_second_step_does_not_rearm_pending_ next_logits[0, 7] = 30.0 next_logits[0, 8] = -2.0 next_logits[0, 10] = 20.0 + original_next_logits = next_logits.clone() model.prepare_duplex_sampling(next_logits, SimpleNamespace(), (row,)) assert state.pending_speech_context is False assert state.current_turn_ended is False - assert torch.isneginf(next_logits[0, 7]) - assert next_logits[0, 8].item() == 30.0 + assert torch.equal(next_logits, original_next_logits) -def test_minicpmo_model_hook_mid_turn_speech_redirects_listen_to_tts_bos(): +def test_minicpmo_model_hook_mid_turn_speech_preserves_logits_for_post_sample_redirect(): state = SimpleNamespace( current_turn_ended=False, last_terminator_token=None, @@ -296,12 +296,11 @@ def test_minicpmo_model_hook_mid_turn_speech_redirects_listen_to_tts_bos(): logits[0, 7] = 30.0 logits[0, 8] = -2.0 logits[0, 10] = 20.0 + original_logits = logits.clone() model.prepare_duplex_sampling(logits, SimpleNamespace(), (row,)) - assert torch.isneginf(logits[0, 7]) - assert logits[0, 8].item() == 30.0 - assert logits[0, 10].item() == 20.0 + assert torch.equal(logits, original_logits) def test_minicpmo_model_hook_ignores_serving_new_user_turn_marker(): @@ -317,12 +316,12 @@ def test_minicpmo_model_hook_ignores_serving_new_user_turn_marker(): logits = torch.zeros((1, 16), dtype=torch.float32) logits[0, 7] = 5.0 logits[0, 10] = 20.0 + original_logits = logits.clone() model.prepare_duplex_sampling(logits, SimpleNamespace(), (row,)) assert state.current_turn_ended is False assert state.last_terminator_token is None - assert torch.isneginf(logits[0, 7]) - assert logits[0, 8].item() == 5.0 + assert torch.equal(logits, original_logits) def test_generic_ar_runner_builds_typed_duplex_sampling_rows(): @@ -613,6 +612,180 @@ def test_minicpmo_stage0_rejects_unknown_special_token_fallbacks(): runtime._require_special_token_ids() +def _minicpmo45_tokenizer_stub(): + token_ids = { + token: index + for index, token in enumerate( + ( + "", + "", + "<|listen|>", + "<|speak|>", + "<|tts_bos|>", + "<|tts_eos|>", + "<|tts_pad|>", + "<|chunk_eos|>", + "<|chunk_tts_eos|>", + "<|turn_eos|>", + ), + start=101, + ) + } + return SimpleNamespace( + unk_token_id=0, + convert_tokens_to_ids=lambda token: token_ids.get(token, 0), + encode=lambda token, add_special_tokens=False: ( + [token_ids[token]] if not add_special_tokens and token in token_ids else [0] + ), + ) + + +def _transformers_processor_stub(auto_processor): + class _AutoImageProcessor: + @staticmethod + def register(*_args, **_kwargs): + return None + + return SimpleNamespace( + AutoImageProcessor=_AutoImageProcessor, + AutoProcessor=auto_processor, + ) + + +def test_minicpmo_stage0_loads_processor_from_hf_id(monkeypatch): + import sys + + from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + MiniCPMO45Stage0DuplexRuntime, + ) + + model_id = "openbmb/MiniCPM-o-4_5" + processor = SimpleNamespace(tokenizer=_minicpmo45_tokenizer_stub()) + load_calls = [] + + class _AutoImageProcessor: + @staticmethod + def register(config_class, image_processor_class): + del image_processor_class + if isinstance(config_class, str): + raise AttributeError("'str' object has no attribute '__module__'") + + class _AutoProcessor: + @classmethod + def from_pretrained(cls, model_path, *, trust_remote_code): + load_calls.append((model_path, trust_remote_code)) + _AutoImageProcessor.register("MiniCPMVImageProcessor", object) + return processor + + monkeypatch.setitem( + sys.modules, + "transformers", + SimpleNamespace( + AutoImageProcessor=_AutoImageProcessor, + AutoProcessor=_AutoProcessor, + ), + ) + + loaded = MiniCPMO45Stage0DuplexRuntime._load_processor_from_path(model_id) + + assert loaded is processor + assert load_calls == [(model_id, True)] + with pytest.raises(AttributeError, match="__module__"): + _AutoImageProcessor.register("MiniCPMVImageProcessor", object) + + +def test_minicpmo_stage0_processor_load_preserves_original_exception(monkeypatch): + import sys + + from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + MiniCPMO45Stage0DuplexRuntime, + ) + + original = ValueError("invalid processor config") + + class _AutoProcessor: + @classmethod + def from_pretrained(cls, model_path, *, trust_remote_code): + del cls, model_path, trust_remote_code + raise original + + monkeypatch.setitem( + sys.modules, + "transformers", + _transformers_processor_stub(_AutoProcessor), + ) + + with pytest.raises(RuntimeError, match="Failed to load MiniCPM-o duplex processor") as exc_info: + MiniCPMO45Stage0DuplexRuntime._load_processor_from_path("openbmb/MiniCPM-o-4_5") + + assert exc_info.value.__cause__ is original + + +def test_minicpmo_stage0_processor_requires_tokenizer(monkeypatch): + import sys + + from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + MiniCPMO45Stage0DuplexRuntime, + ) + + class _AutoProcessor: + @classmethod + def from_pretrained(cls, model_path, *, trust_remote_code): + del cls, model_path, trust_remote_code + return object() + + monkeypatch.setitem( + sys.modules, + "transformers", + _transformers_processor_stub(_AutoProcessor), + ) + + with pytest.raises(RuntimeError, match="does not expose a tokenizer"): + MiniCPMO45Stage0DuplexRuntime._load_processor_from_path("openbmb/MiniCPM-o-4_5") + + +def test_minicpmo_stage0_loaded_processor_validates_special_tokens(monkeypatch): + import sys + + from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + MiniCPMO45Stage0DuplexRuntime, + ) + + processor = SimpleNamespace(tokenizer=_minicpmo45_tokenizer_stub()) + + class _AutoProcessor: + @classmethod + def from_pretrained(cls, model_path, *, trust_remote_code): + del cls, model_path, trust_remote_code + return processor + + monkeypatch.setitem( + sys.modules, + "transformers", + _transformers_processor_stub(_AutoProcessor), + ) + + runtime = MiniCPMO45Stage0DuplexRuntime( + SimpleNamespace(), + model_path="openbmb/MiniCPM-o-4_5", + device="cpu", + ) + + assert runtime.processor is processor + assert set(runtime._special_token_ids()) == { + "unit_token_id", + "unit_end_token_id", + "listen_token_id", + "speak_token_id", + "tts_bos_token_id", + "tts_eos_token_id", + "tts_pad_token_id", + "chunk_eos_token_id", + "chunk_tts_eos_token_id", + "turn_eos_token_id", + } + + def test_minicpmo_stage0_data_plane_prefill_matches_official_unit_format(): import torch @@ -1150,6 +1323,88 @@ def test_minicpmo_stage0_data_plane_final_first_chunk_does_not_add_silence_unit( ) +def test_minicpmo_stage0_final_does_not_promote_first_chunk_alignment_tail_to_unit(): + import torch + + from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( + MiniCPMO45Stage0DuplexRuntime, + _MiniCPMO45Stage0SessionState, + ) + + class _StageModel(torch.nn.Module): + first_chunk_ms = 10 + chunk_ms = 8 + sample_rate = 1000 + + def __init__(self): + super().__init__() + self.embed = torch.nn.Embedding(256, 2) + + def get_input_embeddings(self): + return self.embed + + def get_audio_hidden_states(self, _data): + return [torch.tensor([[0.5, 0.5]], dtype=torch.float32)] + + class _MelProcessor: + sample_rate = 1000 + is_first = True + + def get_config(self): + return {"effective_first_chunk_ms": 9} + + class _Processor: + def __init__(self): + self._streaming_mel_processor = _MelProcessor() + + def get_streaming_chunk_size(self): + return 9 if self._streaming_mel_processor.is_first else 8 + + def process_audio_streaming(self, audio, **_kwargs): + self._streaming_mel_processor.is_first = False + return {"audio_features": audio, "audio_feature_lens": [[len(audio)]]} + + runtime = MiniCPMO45Stage0DuplexRuntime.__new__(MiniCPMO45Stage0DuplexRuntime) + runtime.stage_model = _StageModel() + runtime.thinker = runtime.stage_model + runtime.tokenizer = SimpleNamespace( + unk_token_id=0, + convert_tokens_to_ids=lambda token: { + "": 1, + "": 2, + "<|listen|>": 3, + "<|speak|>": 4, + "<|tts_bos|>": 5, + "<|tts_eos|>": 6, + "<|tts_pad|>": 7, + "<|chunk_eos|>": 8, + "<|chunk_tts_eos|>": 9, + "<|turn_eos|>": 10, + "<|audio|>": 11, + }.get(token, 0), + encode=lambda text, add_special_tokens=False: [], + ) + runtime.processor = _Processor() + runtime.device = "cpu" + runtime._init_token_ids() + state = _MiniCPMO45Stage0SessionState(session_id="sid-first-chunk-alignment-tail") + + first = runtime._stage_prefill_embeddings_only(state, np.zeros(8, dtype=np.float32), seq=1) + second = runtime._stage_prefill_embeddings_only(state, np.zeros(8, dtype=np.float32), seq=2) + final = runtime._stage_prefill_embeddings_only( + state, + np.zeros(8, dtype=np.float32), + seq=3, + final=True, + ) + + assert first["input_token_ids"].count(1) == 1 + assert second["input_token_ids"].count(1) == 1 + assert final["input_token_ids"].count(1) == 1 + assert state.audio_chunk_idx == 3 + assert len(state.audio_buffer) == 1 + + def test_minicpmo_stage0_runtime_uses_loaded_vllm_embed_tokens_when_get_input_embeddings_is_broken(): import torch @@ -1319,7 +1574,7 @@ def test_minicpmo_stage0_native_sampler_penalizes_text_from_prior_chunk(): model.thinker = SimpleNamespace(get_tokenizer=lambda: _Tokenizer()) session_key = ("sid-cross-chunk-repetition", 0) state = _MiniCPMO45Stage0SessionState(session_id=session_key[0]) - state.generated_text_tokens = [198] * 8 + state.generated_tokens = [198] * 8 model._minicpmo45_duplex_data_plane_helper = SimpleNamespace(sessions={session_key: state}) model._minicpmo45_duplex_row_sessions = {0: session_key} @@ -1381,7 +1636,7 @@ def test_minicpmo_stage0_native_sampler_matches_official_negative_logit_penalty( state = _MiniCPMO45Stage0SessionState(session_id=session_key[0]) repeated = 198 alternative = 1234 - state.generated_text_tokens = [repeated] + state.generated_tokens = [repeated] model._minicpmo45_duplex_data_plane_helper = SimpleNamespace(sessions={session_key: state}) model._minicpmo45_duplex_row_sessions = {0: session_key} @@ -1405,7 +1660,7 @@ def test_minicpmo_stage0_native_sampler_matches_official_negative_logit_penalty( assert sampled.sampled_token_ids.tolist() == [[repeated]] -def test_minicpmo_stage0_records_only_bounded_text_repetition_history(): +def test_minicpmo_stage0_records_bounded_model_policy_history(): from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import ( _MiniCPMO45Stage0SessionState, ) @@ -1432,15 +1687,15 @@ def test_minicpmo_stage0_records_only_bounded_text_repetition_history(): } for sampled in range(1000, 1520): - model._record_minicpmo45_duplex_terminator(0, sampled, token_ids) + model._record_minicpmo45_duplex_generation_token(0, sampled) expected_history = list(range(1008, 1520)) - assert state.generated_text_tokens == expected_history + assert state.generated_tokens == expected_history for sampled in token_ids.values(): - model._record_minicpmo45_duplex_terminator(0, sampled, token_ids) + model._record_minicpmo45_duplex_generation_token(0, sampled) - assert state.generated_text_tokens == expected_history + assert state.generated_tokens == [*expected_history[len(token_ids) :], *token_ids.values()] def test_minicpmo_stage0_native_sampler_does_not_override_model_at_punctuation(): diff --git a/vllm_omni/core/sched/omni_ar_scheduler.py b/vllm_omni/core/sched/omni_ar_scheduler.py index d87fc0c8a..b043f0c0b 100644 --- a/vllm_omni/core/sched/omni_ar_scheduler.py +++ b/vllm_omni/core/sched/omni_ar_scheduler.py @@ -801,9 +801,17 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): if self.log_stats: session.record_event(EngineCoreEventType.QUEUED) return - update_info = getattr(update, "additional_information", None) - update_meta = update_info.get("meta") if isinstance(update_info, dict) else None - if isinstance(update_meta, dict) and update_meta.get("replace_streaming_prompt") is True: + update_infos = ( + getattr(update, "model_intermediate_buffer", None), + getattr(update, "additional_information", None), + ) + replace_streaming_prompt = any( + isinstance(info, dict) + and isinstance(info.get("meta"), dict) + and info["meta"].get("replace_streaming_prompt") is True + for info in update_infos + ) + if replace_streaming_prompt: self._replace_streaming_session(session, update) return super()._update_request_as_session(session, update) diff --git a/vllm_omni/core/sched/omni_generation_scheduler.py b/vllm_omni/core/sched/omni_generation_scheduler.py index 26e2cfb83..69c6c3c5c 100644 --- a/vllm_omni/core/sched/omni_generation_scheduler.py +++ b/vllm_omni/core/sched/omni_generation_scheduler.py @@ -56,6 +56,21 @@ class OmniGenerationScheduler(OmniSchedulerMixin, VLLMScheduler): ) self._latest_omni_connector_output: OmniConnectorOutput | None = None + def _handle_stopped_request(self, request: Request) -> bool: + if ( + request.resumable + and not request.streaming_queue + and self.chunk_transfer_adapter is not None + and self.chunk_transfer_adapter.receives_chunks + ): + # Downstream async-chunk stages receive the next segment from the + # connector, not from an API StreamingUpdate. Enqueue them as + # schedulable before the base class can park them in skipped_waiting. + request.status = RequestStatus.WAITING + self._enqueue_waiting_request(request) + return False + return super()._handle_stopped_request(request) + def schedule(self, throttle_prefills: bool = False) -> SchedulerOutput: """One-shot generation fast path: - Feed all input tokens of the request at once diff --git a/vllm_omni/core/sched/omni_scheduler_mixin.py b/vllm_omni/core/sched/omni_scheduler_mixin.py index 3df47a357..684b1f260 100644 --- a/vllm_omni/core/sched/omni_scheduler_mixin.py +++ b/vllm_omni/core/sched/omni_scheduler_mixin.py @@ -63,6 +63,11 @@ class OmniSchedulerMixin: session.num_computed_tokens = 0 session.prompt_token_ids = new_prompt session.additional_information = update.additional_information or None + session.model_intermediate_buffer = getattr( + update, + "model_intermediate_buffer", + None, + ) session.update_block_hashes() session.num_prompt_tokens = len(new_prompt) session.arrival_time = update.arrival_time diff --git a/vllm_omni/deploy/minicpmo_4_5_duplex.yaml b/vllm_omni/deploy/minicpmo_4_5_duplex.yaml index e58c515c7..d8ae8aebe 100644 --- a/vllm_omni/deploy/minicpmo_4_5_duplex.yaml +++ b/vllm_omni/deploy/minicpmo_4_5_duplex.yaml @@ -3,7 +3,7 @@ # The registered minicpmo_4_5 pipeline owns the fixed # Thinker -> Talker -> Code2Wav topology. This overlay changes session # lifetime and bounds all three stages to two live duplex sessions. -base_config: minicpmo_4_5_batching.yaml +base_config: minicpmo_4_5.yaml pipeline: minicpmo_4_5 session_mode: duplex active_stream_window: 1 @@ -31,6 +31,7 @@ stages: # segment and turn boundaries travel as explicit output metadata. default_sampling_params: max_tokens: 2048 + min_tokens: 0 stop_token_ids: [1] - stage_id: 2 diff --git a/vllm_omni/distributed/omni_connectors/adapter.py b/vllm_omni/distributed/omni_connectors/adapter.py index c124e8e43..f1e224b15 100644 --- a/vllm_omni/distributed/omni_connectors/adapter.py +++ b/vllm_omni/distributed/omni_connectors/adapter.py @@ -231,18 +231,18 @@ def construct_next_stage_streaming_input_prompt(payload_data: dict[str, Any], re ids = payload_data.get("ids", {}) meta = payload_data.get("meta", {}) replace_prompt = isinstance(meta, dict) and meta.get("replace_streaming_prompt") is True - condition_len = meta.get("next_stage_prompt_len") if replace_prompt else None - if isinstance(condition_len, int) and condition_len > 0: + next_stage_prompt_len = meta.get("next_stage_prompt_len") if isinstance(meta, dict) else None + if replace_prompt and isinstance(next_stage_prompt_len, int) and next_stage_prompt_len > 0: # Some downstream stages consume complete, independently conditioned # segments instead of extending an existing KV prefix. The producer # declares that transport behavior explicitly in payload metadata. - new_prompt = [0] * condition_len + new_prompt = [0] * next_stage_prompt_len request._output_token_ids.clear() request._all_token_ids.clear() request._all_token_ids.extend(new_prompt) request.prompt_token_ids = new_prompt request.num_computed_tokens = 0 - request.num_prompt_tokens = condition_len + request.num_prompt_tokens = next_stage_prompt_len request.update_block_hashes() return prompt_token_ids = ids.get("prompt", None) @@ -255,7 +255,10 @@ def construct_next_stage_streaming_input_prompt(payload_data: dict[str, Any], re assert request.prompt_token_ids is not None # Extend prompt with kept output tokens. request.prompt_token_ids.extend(kept_output_tokens) - next_prompt_len = max(1, compute_talker_prompt_ids_length(prompt_token_ids)) + if isinstance(next_stage_prompt_len, int) and next_stage_prompt_len > 0: + next_prompt_len = next_stage_prompt_len + else: + next_prompt_len = max(1, compute_talker_prompt_ids_length(prompt_token_ids)) new_prompt = [0] * next_prompt_len request._all_token_ids.extend(new_prompt or ()) request.prompt_token_ids.extend(new_prompt or ()) diff --git a/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py b/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py index 54d1477a9..5005ec280 100644 --- a/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py +++ b/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py @@ -283,7 +283,11 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): # Empty chunk with more data expected: keep polling. has_new_ids = bool(new_ids.numel()) if use_tensor_codes else bool(new_ids) - if not has_new_ids and not payload_finished: + if not has_new_ids and payload_segment_finished: + # Preserve an explicit scheduler boundary even when it + # contains no new codec frames. + request.prompt_token_ids = [0] + if not has_new_ids and not payload_finished and not payload_segment_finished: # The base recv loop treats False as "not ready yet" and # requeues the request. Do not mark an empty non-terminal # chunk as ready, otherwise Stage1 can consume before the @@ -338,7 +342,8 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): if payload_data.meta is None: payload_data.meta = MetaStruct() payload_data.meta.finished = torch.tensor(is_finished, dtype=torch.bool) - payload_data.meta.is_segment_finished = torch.tensor(is_segment_finished, dtype=torch.bool) + if payload_data.meta.is_segment_finished is None: + payload_data.meta.is_segment_finished = torch.tensor(is_segment_finished, dtype=torch.bool) success, size, metadata = self.connector.put( from_stage=str(stage_id), @@ -736,6 +741,14 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): queue_snapshot = list(queue) for request in queue_snapshot: if not self._ensure_active_stream(request): + if target_status == RequestStatus.WAITING: + # A non-active placeholder must not remain visible to the + # scheduler: it has no connector payload yet, so running + # it would execute the downstream model with empty + # additional_information. Park it until restore_queues() + # and retry admission on the next scheduler tick. + queue.remove(request) + waiting_for_chunk_list.append(request) continue if request.status != RequestStatus.WAITING_FOR_CHUNK: if request.request_id in self.requests_with_ready_chunks: diff --git a/vllm_omni/engine/async_omni_engine.py b/vllm_omni/engine/async_omni_engine.py index 17bee5d03..a01d905b1 100644 --- a/vllm_omni/engine/async_omni_engine.py +++ b/vllm_omni/engine/async_omni_engine.py @@ -209,13 +209,13 @@ class AsyncOmniEngine: self._omni_master_port, ) - # Stage resolution pops deploy_config, so get the pipeline endpoint - # restriction beforehand. TODO (Alex) make this cleaner and refactor - # stage config resolution to remove kwargs hacks. - deploy_config_path = kwargs.get("deploy_config") + # Stage resolution pops deploy_config, so get pipeline-wide settings + # beforehand. The stage CLI exposes the same deploy YAML through + # stage_configs_path. + deploy_config_path = kwargs.get("deploy_config") or kwargs.get("stage_configs_path") pipeline_config = StageConfigFactory.get_pipeline_config( model=model, - trust_remote_code=trust_remote_code, + trust_remote_code=bool(trust_remote_code), deploy_config_path=deploy_config_path, ) self.endpoint_restrictions = pipeline_config.endpoint_restrictions if pipeline_config is not None else () diff --git a/vllm_omni/engine/orchestrator.py b/vllm_omni/engine/orchestrator.py index fa10653be..53f30ea48 100644 --- a/vllm_omni/engine/orchestrator.py +++ b/vllm_omni/engine/orchestrator.py @@ -13,7 +13,7 @@ from __future__ import annotations import asyncio import time as _time -from collections.abc import Callable +from collections.abc import Awaitable, Callable from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any @@ -218,11 +218,18 @@ class _OrchestratorDuplexStagePort: request_states: dict[str, OrchestratorRequestState], running_counter: OmniRequestCounter | None, cleanup_request_ids: Callable[..., Any], + async_chunk: bool, + prewarm_async_chunk_stages: Callable[ + [str, Any, OrchestratorRequestState], + Awaitable[None], + ], ) -> None: self._stage_pools = stage_pools self._request_states = request_states self._running_counter = running_counter self._cleanup_request_ids = cleanup_request_ids + self._async_chunk = async_chunk + self._prewarm_async_chunk_stages = prewarm_async_chunk_stages @property def stage_count(self) -> int: @@ -299,6 +306,12 @@ class _OrchestratorDuplexStagePort: replica_id = await pool.submit_update(context.request_id, request_state, request) else: replica_id = await pool.submit_initial(context.request_id, request_state, request, prompt_text=None) + if self._async_chunk and context.stage_id == 0: + await self._prewarm_async_chunk_stages( + context.request_id, + request, + request_state, + ) request_state.duplex_stage_fences[context.stage_id] = context.fence request_state.stage_submit_ts[context.stage_id] = _time.time() if not request_state.running_counter_registered and self._running_counter is not None: @@ -388,6 +401,8 @@ class Orchestrator: request_states=self.request_states, running_counter=self._running_counter, cleanup_request_ids=self._cleanup_request_ids, + async_chunk=self.async_chunk, + prewarm_async_chunk_stages=self._prewarm_async_chunk_stages, ), result_sink=self.rpc_async_queue, lifecycle_sink=self.output_async_queue, @@ -2017,7 +2032,7 @@ class Orchestrator: _t_submit_start = _time.perf_counter() if next_pool.stage_type == "diffusion": - await next_pool.submit_initial( + replica_id = await next_pool.submit_initial( request_id, req_state, req_state.prompt, @@ -2056,12 +2071,18 @@ class Orchestrator: resumable=downstream_resumable, ) request.external_req_id = request.request_id - await next_pool.submit_initial( + replica_id = await next_pool.submit_initial( request_id, req_state, request, prompt_text=None, ) + self._record_duplex_stage_submission( + next_stage_id, + request_id, + replica_id, + req_state, + ) # async_chunk pre-submit fires per stage edge (N-1 -> N). Source # replica is stage 0's bound replica (single-replica thinker in diff --git a/vllm_omni/engine/stage_engine_startup.py b/vllm_omni/engine/stage_engine_startup.py index d23d9f3e0..cb80c82ac 100644 --- a/vllm_omni/engine/stage_engine_startup.py +++ b/vllm_omni/engine/stage_engine_startup.py @@ -183,6 +183,10 @@ class OmniMasterServer: # seeded with the registration port (and coordinator ROUTER port) that # are already bound on the same host. self._allocated_ports: set[int] = set() + # Keep route ports bound until their ZMQ owners are ready to bind them. + # Recording only the integer leaves a race with unrelated startup + # consumers (notably NCCL's own ephemeral-port selection). + self._port_reservations: dict[int, socket.socket] = {} if master_port: self._allocated_ports.add(int(master_port)) # Coordinator ROUTER address echoed back in every registration reply @@ -283,9 +287,11 @@ class OmniMasterServer: ``get_open_ports_list`` dedups only within its own call, so we redraw any port that collides with one already recorded in ``self._allocated_ports`` (registration/coordinator ports plus every previously-allocated route - port) and register the winners before returning. Callers already hold the - relevant allocation context (``__init__`` is single-threaded; - registration holds ``self._alloc_lock``). + port). Each winner remains OS-bound until its ZMQ owner is ready, so + another startup subsystem cannot claim it in the allocation-to-bind + window. Callers already hold the relevant allocation context + (``__init__`` is single-threaded; registration holds + ``self._alloc_lock``). """ picked: list[int] = [] # Bounded retry budget so a pathological host (near port exhaustion) @@ -294,7 +300,17 @@ class OmniMasterServer: for port in get_open_ports_list(count=count - len(picked)): if port in self._allocated_ports: continue + reservation = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + # Reserve on every local interface. The advertised address + # can be a routable interface IP, but wildcard binding also + # protects the route from local NCCL/socket allocations. + reservation.bind(("", port)) + except OSError: + reservation.close() + continue self._allocated_ports.add(port) + self._port_reservations[port] = reservation picked.append(port) if len(picked) == count: return picked @@ -303,6 +319,33 @@ class OmniMasterServer: f"(host={self._address}); {len(self._allocated_ports)} already in use." ) + def release_route_port_reservations( + self, + stage_id: int, + replica_id: int = 0, + *, + handshake: bool = True, + data: bool = True, + ) -> None: + """Release reserved route ports immediately before their ZMQ binds.""" + alloc = self.get_allocation(stage_id, replica_id) + ports: list[int] = [] + if handshake: + handshake_port = _port_from_zmq_address(alloc.handshake_bind_address) + if handshake_port is not None: + ports.append(handshake_port) + if data: + for address in (alloc.input_bind_address, alloc.output_bind_address): + port = _port_from_zmq_address(address) + if port is not None: + ports.append(port) + + with self._alloc_lock: + for port in ports: + reservation = self._port_reservations.pop(port, None) + if reservation is not None: + reservation.close() + def _next_free_replica_id(self, stage_id: int) -> int: """Return the next replica id to assign for an auto-assign registration. @@ -437,6 +480,11 @@ class OmniMasterServer: self._stop_event.set() if self._thread is not None: self._thread.join(timeout=10) + with self._alloc_lock: + reservations = list(self._port_reservations.values()) + self._port_reservations.clear() + for reservation in reservations: + reservation.close() # ------------------------------------------------------------------ # Internal server logic @@ -771,20 +819,34 @@ def connect_remote_engine_cores( handshake_bind_address = omni_master_server.get_allocation(stage_id, replica_id=replica_id).handshake_bind_address - with zmq_socket_ctx(handshake_bind_address, zmq.ROUTER, bind=True) as handshake_socket: - yield StageReplicaResources( - coordinator=coordinator, - addresses=addresses, - ) - wait_for_engine_startup( - handshake_socket, - addresses, - engines_to_handshake, - vllm_config.parallel_config, - False, # coordinated_dp - vllm_config.cache_config, - None, # proc_manager (remote — no local procs) - None, # coord_process + omni_master_server.release_route_port_reservations( + stage_id, + replica_id, + handshake=True, + data=False, + ) + try: + with zmq_socket_ctx(handshake_bind_address, zmq.ROUTER, bind=True) as handshake_socket: + yield StageReplicaResources( + coordinator=coordinator, + addresses=addresses, + ) + wait_for_engine_startup( + handshake_socket, + addresses, + engines_to_handshake, + vllm_config.parallel_config, + False, # coordinated_dp + vllm_config.cache_config, + None, # proc_manager (remote — no local procs) + None, # coord_process + ) + finally: + omni_master_server.release_route_port_reservations( + stage_id, + replica_id, + handshake=False, + data=True, ) @@ -802,17 +864,31 @@ def connect_remote_diffusion_proc( ).handshake_bind_address logger.info("Waiting for remote diffusion proc for stage %d replica %d", stage_id, replica_id) - with zmq_socket_ctx(handshake_bind_address, zmq.ROUTER, bind=True) as handshake_socket: - yield StageReplicaResources(addresses=addresses) - wait_for_engine_startup( - handshake_socket, - addresses, - [CoreEngine(index=0, local=False)], - _single_diffusion_parallel_config(local_client=False), - False, - None, - None, - None, + omni_master_server.release_route_port_reservations( + stage_id, + replica_id, + handshake=True, + data=False, + ) + try: + with zmq_socket_ctx(handshake_bind_address, zmq.ROUTER, bind=True) as handshake_socket: + yield StageReplicaResources(addresses=addresses) + wait_for_engine_startup( + handshake_socket, + addresses, + [CoreEngine(index=0, local=False)], + _single_diffusion_parallel_config(local_client=False), + False, + None, + None, + None, + ) + finally: + omni_master_server.release_route_port_reservations( + stage_id, + replica_id, + handshake=False, + data=True, ) @@ -918,49 +994,63 @@ def _launch_omni_core_engines( # Bind the pre-allocated handshake socket for this stage. handshake_bind_address = omni_master_server.get_allocation(stage_id, replica_id=replica_id).handshake_bind_address - with zmq_socket_ctx(handshake_bind_address, zmq.ROUTER, bind=True) as handshake_socket: - if omni_coordinator_address is not None: - # Use the omni subclass so each spawned subprocess instantiates - # an OmniCoordClientForStage and heartbeats to the coordinator. - from vllm_omni.engine.stage_engine_core_proc_manager import StageEngineCoreProcManager + omni_master_server.release_route_port_reservations( + stage_id, + replica_id, + handshake=True, + data=False, + ) + try: + with zmq_socket_ctx(handshake_bind_address, zmq.ROUTER, bind=True) as handshake_socket: + if omni_coordinator_address is not None: + # Use the omni subclass so each spawned subprocess instantiates + # an OmniCoordClientForStage and heartbeats to the coordinator. + from vllm_omni.engine.stage_engine_core_proc_manager import StageEngineCoreProcManager - with scoped_spawn_device_env(stage_visible_devices, spawn_device_lock): - local_engine_manager: CoreEngineProcManager = StageEngineCoreProcManager( - local_engine_count=local_engine_count, - start_index=start_index, - local_start_index=local_start_index, - vllm_config=vllm_config, - local_client=True, - handshake_address=handshake_address, - executor_class=executor_class, - log_stats=log_stats, - omni_stage_id=stage_id, - omni_coordinator_address=omni_coordinator_address, - omni_replica_base_id=replica_id, - ) - else: - with scoped_spawn_device_env(stage_visible_devices, spawn_device_lock): - local_engine_manager = CoreEngineProcManager( - local_engine_count=local_engine_count, - start_index=start_index, - local_start_index=local_start_index, - vllm_config=vllm_config, - local_client=True, - handshake_address=handshake_address, - executor_class=executor_class, - log_stats=log_stats, - ) + with scoped_spawn_device_env(stage_visible_devices, spawn_device_lock): + local_engine_manager: CoreEngineProcManager = StageEngineCoreProcManager( + local_engine_count=local_engine_count, + start_index=start_index, + local_start_index=local_start_index, + vllm_config=vllm_config, + local_client=True, + handshake_address=handshake_address, + executor_class=executor_class, + log_stats=log_stats, + omni_stage_id=stage_id, + omni_coordinator_address=omni_coordinator_address, + omni_replica_base_id=replica_id, + ) + else: + with scoped_spawn_device_env(stage_visible_devices, spawn_device_lock): + local_engine_manager = CoreEngineProcManager( + local_engine_count=local_engine_count, + start_index=start_index, + local_start_index=local_start_index, + vllm_config=vllm_config, + local_client=True, + handshake_address=handshake_address, + executor_class=executor_class, + log_stats=log_stats, + ) - yield local_engine_manager, coordinator, addresses - wait_for_engine_startup( - handshake_socket, - addresses, - engines_to_handshake, - parallel_config, - parallel_config.data_parallel_size > 1 and vllm_config.model_config.is_moe, - vllm_config.cache_config, - local_engine_manager, - coordinator.proc if coordinator else None, + yield local_engine_manager, coordinator, addresses + wait_for_engine_startup( + handshake_socket, + addresses, + engines_to_handshake, + parallel_config, + parallel_config.data_parallel_size > 1 and vllm_config.model_config.is_moe, + vllm_config.cache_config, + local_engine_manager, + coordinator.proc if coordinator else None, + ) + finally: + omni_master_server.release_route_port_reservations( + stage_id, + replica_id, + handshake=False, + data=True, ) @@ -1448,6 +1538,12 @@ def launch_diffusion_stage_replica( omni_stage_config=stage_config, replica_id=replica_id, ) + omni_master_server.release_route_port_reservations( + metadata.stage_id, + replica_id, + handshake=True, + data=False, + ) proc_manager = stage_diffusion_proc.StageDiffusionProcManager( model=model, od_config=od_config, @@ -1461,6 +1557,12 @@ def launch_diffusion_stage_replica( omni_stage_id=metadata.stage_id, omni_replica_id=replica_id, ) + omni_master_server.release_route_port_reservations( + metadata.stage_id, + replica_id, + handshake=False, + data=True, + ) client = StageDiffusionClient.from_addresses( metadata, request_address=proc_manager.addresses.inputs[0], @@ -1474,6 +1576,10 @@ def launch_diffusion_stage_replica( lock_fds=lock_fds, ) except Exception: + omni_master_server.release_route_port_reservations( + metadata.stage_id, + replica_id, + ) if proc_manager is not None: proc_manager.shutdown() if lock_fds: diff --git a/vllm_omni/entrypoints/cli/serve.py b/vllm_omni/entrypoints/cli/serve.py index 13ee808be..8f6a88190 100644 --- a/vllm_omni/entrypoints/cli/serve.py +++ b/vllm_omni/entrypoints/cli/serve.py @@ -894,7 +894,7 @@ def run_headless(args: TrackingNamespace) -> None: stage_connector_spec = get_stage_connector_spec( omni_transfer_config=omni_transfer_config, stage_id=stage_id, - async_chunk=False, + async_chunk=bool(stage_cfg.engine_args.get("async_chunk", False)), ) # ``runtime_cfg`` is mostly inherited from the parent's diff --git a/vllm_omni/experimental/fullduplex/minicpmo45/data_plane.py b/vllm_omni/experimental/fullduplex/minicpmo45/data_plane.py index d3b082ffc..d18c0f4a9 100644 --- a/vllm_omni/experimental/fullduplex/minicpmo45/data_plane.py +++ b/vllm_omni/experimental/fullduplex/minicpmo45/data_plane.py @@ -65,12 +65,11 @@ class MiniCPMO45DataPlaneSession: def begin_request(self, request_id: str) -> None: state = self._requests.setdefault(request_id, _RequestState()) state.terminal = False - # Preserve turn-scoped cursors for resumable requests. The legacy path - # only reset request-scoped terminal markers on append. - request_turn = state.turns.get(None) - if request_turn is not None: - request_turn.tts_eos_done = False - request_turn.turn_eos_done = False + # A resumable append starts a new Talker segment, even when it + # continues the same model turn. Re-arm only the segment boundary; + # turn EOS remains deduplicated until the model advances the turn. + for turn_state in state.turns.values(): + turn_state.tts_eos_done = False def is_terminal(self, request_id: str | None) -> bool: if request_id is None: diff --git a/vllm_omni/experimental/fullduplex/minicpmo45/input.py b/vllm_omni/experimental/fullduplex/minicpmo45/input.py index 2a4eac3ea..248e857e8 100644 --- a/vllm_omni/experimental/fullduplex/minicpmo45/input.py +++ b/vllm_omni/experimental/fullduplex/minicpmo45/input.py @@ -2,11 +2,19 @@ from __future__ import annotations import base64 import binascii +from dataclasses import dataclass from typing import Any import numpy as np +@dataclass(frozen=True, slots=True) +class _PcmSpan: + byte_count: int + force_listen: bool + is_speech: bool + + class MiniCPMO45PcmAppendReservation: __slots__ = ( "_active", @@ -15,6 +23,7 @@ class MiniCPMO45PcmAppendReservation: "_owner", "_raw", "_sample_rate_hz", + "_spans", "_turn_had_speech", "_video_frames", "operation_id", @@ -31,6 +40,7 @@ class MiniCPMO45PcmAppendReservation: sample_rate_hz: int, force_listen: bool, is_speech: bool, + spans: list[_PcmSpan] | None = None, turn_had_speech: bool = False, video_frames: list[str] | None = None, ) -> None: @@ -41,6 +51,7 @@ class MiniCPMO45PcmAppendReservation: self._sample_rate_hz = sample_rate_hz self._force_listen = force_listen self._is_speech = is_speech + self._spans = list(spans or []) self._turn_had_speech = turn_had_speech self._video_frames = list(video_frames or []) self._active = True @@ -97,9 +108,8 @@ class MiniCPMO45PcmAppendBuffer: def __init__(self) -> None: self._buffer = bytearray() + self._spans: list[_PcmSpan] = [] self._sample_rate_hz: int | None = None - self._force_listen = False - self._is_speech = False self._turn_had_speech = False self._reservation_seq = 0 self._reservations: list[MiniCPMO45PcmAppendReservation] = [] @@ -112,14 +122,13 @@ class MiniCPMO45PcmAppendBuffer: reservation._active = False self._reservations.clear() self._buffer.clear() + self._spans.clear() self._frame_queue.clear() self._sample_rate_hz = None - self._force_listen = False - self._is_speech = False self._turn_had_speech = False def clear_force_listen(self) -> None: - self._force_listen = False + self._spans = [_PcmSpan(span.byte_count, False, span.is_speech) for span in self._spans] def has_pending(self) -> bool: return bool(self._buffer) @@ -131,6 +140,56 @@ class MiniCPMO45PcmAppendBuffer: def pending_byte_count(self) -> int: return len(self._buffer) + def _append_span(self, span: _PcmSpan) -> None: + if span.byte_count <= 0: + return + if ( + self._spans + and self._spans[-1].force_listen == span.force_listen + and self._spans[-1].is_speech == span.is_speech + ): + previous = self._spans[-1] + self._spans[-1] = _PcmSpan( + previous.byte_count + span.byte_count, + span.force_listen, + span.is_speech, + ) + return + self._spans.append(span) + + def _consume_spans(self, byte_count: int) -> list[_PcmSpan]: + consumed: list[_PcmSpan] = [] + remaining = byte_count + while remaining: + if not self._spans: + raise RuntimeError("MiniCPM-o PCM metadata is shorter than buffered audio") + span = self._spans.pop(0) + take = min(remaining, span.byte_count) + consumed.append(_PcmSpan(take, span.force_listen, span.is_speech)) + if take < span.byte_count: + self._spans.insert( + 0, + _PcmSpan( + span.byte_count - take, + span.force_listen, + span.is_speech, + ), + ) + remaining -= take + return consumed + + def _prepend_spans(self, spans: list[_PcmSpan]) -> None: + current = self._spans + self._spans = [] + for span in [*spans, *current]: + self._append_span(span) + + def _buffer_force_listen(self) -> bool: + return any(span.force_listen for span in self._spans) + + def _buffer_is_speech(self) -> bool: + return any(span.is_speech for span in self._spans) + def _reserve_passthrough( self, payload: dict[str, object], @@ -180,12 +239,17 @@ class MiniCPMO45PcmAppendBuffer: raise ValueError("MiniCPM-o native duplex audio append sample_rate_hz changed within a session") self._sample_rate_hz = sample_rate_hz self._buffer.extend(raw) + self._append_span( + _PcmSpan( + len(raw), + bool(payload.get("force_listen", False)), + bool(payload.get("is_speech", False)), + ) + ) frames_in = payload.get("video_frames") if isinstance(frames_in, list): self._frame_queue.extend(frame for frame in frames_in if isinstance(frame, str) and frame) self._turn_had_speech = self._turn_had_speech or bool(payload.get("is_speech", False)) - self._force_listen = self._force_listen or bool(payload.get("force_listen", False)) - self._is_speech = self._is_speech or bool(payload.get("is_speech", False)) if not allow_emit: return None @@ -208,6 +272,7 @@ class MiniCPMO45PcmAppendBuffer: pad_samples = 0 emit_bytes = emit_samples * 4 reserved_raw = bytes(self._buffer[:emit_bytes]) + reserved_spans = self._consume_spans(emit_bytes) emit_raw = reserved_raw + b"\x00" * (pad_samples * 4) del self._buffer[:emit_bytes] @@ -227,11 +292,8 @@ class MiniCPMO45PcmAppendBuffer: if emit_samples + pad_samples >= min_samples and self._frame_queue: attached_frames = [self._frame_queue.pop(0)] out["video_frames"] = attached_frames - out["force_listen"] = self._force_listen - out["is_speech"] = self._is_speech - if not self._buffer: - self._force_listen = False - self._is_speech = False + out["force_listen"] = any(span.force_listen for span in reserved_spans) + out["is_speech"] = any(span.is_speech for span in reserved_spans) reservation = MiniCPMO45PcmAppendReservation( owner=self, operation_id=operation_id, @@ -240,6 +302,7 @@ class MiniCPMO45PcmAppendBuffer: sample_rate_hz=sample_rate_hz, force_listen=bool(out.get("force_listen", False)), is_speech=bool(out.get("is_speech", False)), + spans=reserved_spans, turn_had_speech=self._turn_had_speech, video_frames=attached_frames, ) @@ -261,8 +324,8 @@ class MiniCPMO45PcmAppendBuffer: "audio": "", "format": "pcm_f32le", "sample_rate_hz": self._sample_rate_hz or 16000, - "force_listen": self._force_listen, - "is_speech": self._is_speech, + "force_listen": self._buffer_force_listen(), + "is_speech": self._buffer_is_speech(), } reservation = self.prepare_append( payload, @@ -280,8 +343,8 @@ class MiniCPMO45PcmAppendBuffer: payload=None, raw=b"", sample_rate_hz=self._sample_rate_hz or 0, - force_listen=self._force_listen, - is_speech=self._is_speech, + force_listen=self._buffer_force_listen(), + is_speech=self._buffer_is_speech(), turn_had_speech=had_speech, ) self._reservations.append(reservation) @@ -289,8 +352,6 @@ class MiniCPMO45PcmAppendBuffer: # Open the next physical input generation without touching already # reserved appends. A rollback restores this generation's metadata. self._sample_rate_hz = None - self._force_listen = False - self._is_speech = False self._turn_had_speech = False return reservation @@ -335,12 +396,11 @@ class MiniCPMO45PcmAppendBuffer: rolled_back = self._reservations[index:] restored = b"".join(item._raw for item in rolled_back) self._buffer[:0] = restored + self._prepend_spans([span for item in rolled_back for span in item._spans]) restored_frames = [frame for item in rolled_back for frame in item._video_frames] if restored_frames: self._frame_queue[:0] = restored_frames self._sample_rate_hz = self._sample_rate_hz or reservation._sample_rate_hz - self._force_listen = self._force_listen or any(item._force_listen for item in rolled_back) - self._is_speech = self._is_speech or any(item._is_speech for item in rolled_back) self._turn_had_speech = self._turn_had_speech or any(item._turn_had_speech for item in rolled_back) for item in rolled_back: item._active = False @@ -354,8 +414,8 @@ class MiniCPMO45PcmAppendBuffer: "audio": "", "format": "pcm_f32le", "sample_rate_hz": self._sample_rate_hz or 16000, - "force_listen": self._force_listen, - "is_speech": self._is_speech, + "force_listen": self._buffer_force_listen(), + "is_speech": self._buffer_is_speech(), } return self.append(payload, chunk_period_ms=chunk_period_ms, flush=True) diff --git a/vllm_omni/experimental/fullduplex/minicpmo45/stage0.py b/vllm_omni/experimental/fullduplex/minicpmo45/stage0.py index 84aaf1cb8..8a304d101 100644 --- a/vllm_omni/experimental/fullduplex/minicpmo45/stage0.py +++ b/vllm_omni/experimental/fullduplex/minicpmo45/stage0.py @@ -2,10 +2,10 @@ from __future__ import annotations import base64 import copy -import sys import time from contextlib import suppress from dataclasses import dataclass, field +from threading import Lock from typing import Any import numpy as np @@ -14,6 +14,7 @@ from vllm_omni.experimental.fullduplex.minicpmo45.policy import MiniCPMO45Duplex _MINICPMO45_SPECIAL_TOKEN_FIELDS = MiniCPMO45DuplexPolicy.SPECIAL_TOKEN_FIELDS _MINICPMO45_OPTIONAL_TOKEN_FIELDS = MiniCPMO45DuplexPolicy.OPTIONAL_TOKEN_FIELDS +_MINICPMO45_PROCESSOR_LOAD_LOCK = Lock() @dataclass @@ -35,7 +36,7 @@ class _MiniCPMO45Stage0SessionState: pending_speech_context: bool = False pending_speech_append_identity: tuple[int | None, int] | None = None pending_speech_response_open: bool = False - generated_text_tokens: list[int] = field(default_factory=list) + generated_tokens: list[int] = field(default_factory=list) class MiniCPMO45Stage0DuplexRuntime: @@ -58,6 +59,8 @@ class MiniCPMO45Stage0DuplexRuntime: else getattr(stage_model, "tokenizer", None) ) self._init_token_ids() + if self.tokenizer is not None: + self._require_special_token_ids() def _stage_runtime_ready(self) -> bool: return self.processor is not None and self.tokenizer is not None and self.thinker is not None @@ -196,19 +199,15 @@ class MiniCPMO45Stage0DuplexRuntime: embed_parts.extend(state.context_embeds) token_ids.extend(state.context_token_ids) - # Consume every complete chunk in the buffer so the appended span and - # the scheduler's slot reservation for this append agree exactly. A - # final append may zero-pad a real residual chunk, but it must not add - # a whole silence unit after all input was already consumed: official - # duplex generation runs once per microphone unit and client commit is - # not an additional model decision. + # Consume every complete processor chunk in the buffer so the appended + # span and the scheduler's slot reservation agree exactly. The serving + # PCM buffer pads a final real residual before it reaches Stage0. Do not + # pad again here: the first processor chunk is hop-aligned below 1035ms + # and intentionally leaves a small carry that is not another model unit. units_built = 0 while True: if len(state.audio_buffer) < chunk_size: - if not final or len(state.audio_buffer) == 0: - break - pad = np.zeros(chunk_size - len(state.audio_buffer), dtype=np.float32) - state.audio_buffer = np.concatenate([state.audio_buffer, pad]) + break audio_chunk = state.audio_buffer[:chunk_size] batch_feature = self._process_streaming_audio(audio_chunk, state.audio_chunk_idx, processor=processor) for name, value in ( @@ -224,6 +223,11 @@ class MiniCPMO45Stage0DuplexRuntime: if units_built == 0: return self._stage_prefill_result(False, start_time, "streaming audio embedding returned empty") break + consumed_samples = self._consumed_audio_samples( + state.audio_chunk_idx, + chunk_size, + processor=processor, + ) if state.audio_chunk_idx > 0: # Official duplex closes every unit (finalize_unit feeds the # sampled terminator + ) before the next opens. @@ -255,9 +259,7 @@ class MiniCPMO45Stage0DuplexRuntime: token_ids.extend( [self._audio_embedding_placeholder_token_id()] * int(self._as_2d_tensor(audio_embeds).shape[0]) ) - state.audio_buffer = state.audio_buffer[ - self._consumed_audio_samples(state.audio_chunk_idx, chunk_size, processor=processor) : - ] + state.audio_buffer = state.audio_buffer[consumed_samples:] state.audio_chunk_idx += 1 units_built += 1 chunk_size = self._streaming_chunk_size(processor) @@ -610,14 +612,33 @@ class MiniCPMO45Stage0DuplexRuntime: def _load_processor_from_path(model_path: str | None) -> Any | None: if not model_path: return None - if model_path not in sys.path: - sys.path.insert(0, model_path) try: - from processing_minicpmo import MiniCPMOProcessor + from transformers import AutoImageProcessor, AutoProcessor - return MiniCPMOProcessor.from_pretrained(model_path, trust_remote_code=True) - except Exception: - return None + original_register = AutoImageProcessor.register + + def register_image_processor(config_class, *args, **kwargs): + # The checkpoint's auto_map already loads this class. Its + # legacy string registration is incompatible with some + # Transformers versions and is otherwise redundant. + if config_class == "MiniCPMVImageProcessor": + return None + return original_register(config_class, *args, **kwargs) + + with _MINICPMO45_PROCESSOR_LOAD_LOCK: + AutoImageProcessor.register = staticmethod(register_image_processor) + try: + processor = AutoProcessor.from_pretrained( + model_path, + trust_remote_code=True, + ) + finally: + AutoImageProcessor.register = staticmethod(original_register) + except Exception as exc: + raise RuntimeError(f"Failed to load MiniCPM-o duplex processor from {model_path!r}") from exc + if getattr(processor, "tokenizer", None) is None: + raise RuntimeError(f"MiniCPM-o duplex processor loaded from {model_path!r} does not expose a tokenizer") + return processor def _init_token_ids(self) -> None: if self.tokenizer is None: diff --git a/vllm_omni/experimental/fullduplex/openai/session_runner.py b/vllm_omni/experimental/fullduplex/openai/session_runner.py index 7e2d1ecaf..4b5e4f2e6 100644 --- a/vllm_omni/experimental/fullduplex/openai/session_runner.py +++ b/vllm_omni/experimental/fullduplex/openai/session_runner.py @@ -503,11 +503,45 @@ class DuplexSessionRunnerMixin: clear_completed_pending_silence() pending_silence = native.pending_silence_task if pending_silence is not None and not pending_silence.done(): + if pending_silence is asyncio.current_task(): + return False + try: + if not await pending_silence: + return False + except asyncio.CancelledError: + current = asyncio.current_task() + if current is not None and current.cancelling(): + raise + return False + except Exception: + return False + clear_completed_pending_silence() + pending_silence = native.pending_silence_task + if pending_silence is not None and not pending_silence.done(): + return False + append_tail = actor.native_append_tail + if (append_tail is None or append_tail.done()) and real_native_input_waiting(): return False - if actor.native_append_tail is not None and not actor.native_append_tail.done(): - return False - if real_native_input_waiting(): - return False + continuation_delay_s = max( + 0.0, + float(session.capabilities.chunk_period_ms or 1000) / 1000.0, + ) + if continuation_delay_s > 0: + await asyncio.sleep(continuation_delay_s) + if ( + actor.native_append_tail is not append_tail + or ((append_tail is None or append_tail.done()) and real_native_input_waiting()) + or self._native_silence_continuation_is_stale( + session, + request_id=request_id, + response_id=response_id, + response_owned=response_owned, + expected_epoch=expected_epoch, + expected_incarnation=expected_incarnation, + expected_model_turn_id=expected_model_turn_id, + ) + ): + return False def _still_valid() -> bool: return not real_native_input_waiting() and not self._native_silence_continuation_is_stale( diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py b/vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py index b068b7e4d..f0e07e50b 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py @@ -348,9 +348,16 @@ class BatchedToken2Wav(nn.Module): previous: torch.Tensor, window: torch.Tensor, ) -> torch.Tensor: - overlap = int(window.shape[0] // 2) + overlap = min( + int(window.shape[0] // 2), + int(speech.shape[-1]), + int(previous.shape[-1]), + ) result = speech.clone() - result[..., :overlap] = result[..., :overlap] * window[:overlap] + previous[..., -overlap:] * window[overlap:] + if overlap > 0: + result[..., :overlap] = ( + result[..., :overlap] * window[:overlap] + previous[..., -overlap:] * window[-overlap:] + ) return result def decode_batch( @@ -360,6 +367,7 @@ class BatchedToken2Wav(nn.Module): states: list[BatchedToken2WavState], *, last_chunk: bool, + flush_encoder: bool = False, ) -> tuple[list[torch.Tensor], list[BatchedToken2WavState]]: batch_size = int(tokens.shape[0]) if batch_size != len(states): @@ -382,7 +390,7 @@ class BatchedToken2Wav(nn.Module): with self._autocast(tokens.device): hidden, conformer_cnn, conformer_att = self._encode_chunk( tokens, - last_chunk=last_chunk, + last_chunk=last_chunk or flush_encoder, cnn_cache=flow_cache["conformer_cnn_cache"], att_cache=flow_cache["conformer_att_cache"], ) diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py index 3eac2e49d..92eccb552 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py @@ -4,12 +4,12 @@ from __future__ import annotations -import hashlib import json import os import tempfile from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass +from hashlib import sha256 from pathlib import Path from typing import Any @@ -91,6 +91,13 @@ class _RequestState: token2wav: BatchedToken2WavState +@dataclass +class _RuntimePrompt: + cache_id: str + path: str + owners: set[str] + + @dataclass(frozen=True) class _WorkItem: output_index: int @@ -103,9 +110,11 @@ class _WorkItem: last_chunk: bool tokens: torch.Tensor previous: _RequestState | None - segment_text: str - duplex_turn_id: int | None - duplex_epoch: int | None + runtime_prompt_key: str | None + duplex_epoch: int + duplex_turn_id: int + segment_text_utf8: torch.Tensor + tts_is_last_chunk: bool segment_end: bool turn_end: bool has_payload: bool = True @@ -135,7 +144,11 @@ class MiniCPMO45Code2Wav(nn.Module): self._model_revision = getattr(vllm_config.model_config, "revision", None) self.backend: BatchedToken2Wav | None = None self._states: dict[str, _RequestState] = {} - self._owned_prompt_wavs: dict[str, tuple[str, str]] = {} + self._runtime_prompts: dict[str, _RuntimePrompt] = {} + self._request_prompt_keys: dict[str, str] = {} + self._runtime_prompt_dir = tempfile.TemporaryDirectory( + prefix="minicpmo45-runtime-prompts-", + ) extra = self._extra_config() self._min_batch_size = int(extra.get("code2wav_min_batch_size", 1)) if self._min_batch_size < 1: @@ -158,63 +171,122 @@ class MiniCPMO45Code2Wav(nn.Module): extra = getattr(connector, "extra", None) return dict(extra) if isinstance(extra, Mapping) else {} - def _release_prompt(self, state_id: str) -> None: - owned = self._owned_prompt_wavs.pop(state_id, None) - if owned is None: - return - prompt_cache_id, prompt_wav = owned - if self.backend is not None: - self.backend.evict_prompt(prompt_cache_id, prompt_wav) - try: - os.unlink(prompt_wav) - except FileNotFoundError: - pass - - def _write_reference_prompt( - self, - state_id: str, - reference: object, - sample_rate: object, - ) -> tuple[str, str]: - waveform = torch.as_tensor(reference, dtype=torch.float32).reshape(-1).cpu() - if waveform.numel() == 0: - return self._default_prompt_id, self._default_prompt_wav - sample_rate_hz = int(_scalar(sample_rate, 24000)) - if sample_rate_hz <= 0: - raise _batch_error( - "invalid_ref_audio_sample_rate", - request_id=state_id, - sample_rate=sample_rate_hz, - ) - digest = hashlib.sha256() - digest.update(str(sample_rate_hz).encode("ascii")) - digest.update(waveform.numpy().tobytes()) - prompt_cache_id = f"{state_id}:{digest.hexdigest()}" - current = self._owned_prompt_wavs.get(state_id) - if current is not None and current[0] == prompt_cache_id and os.path.exists(current[1]): - return current - - self._release_prompt(state_id) - handle = tempfile.NamedTemporaryFile(prefix="minicpmo45-ref-", suffix=".wav", delete=False) - prompt_wav = handle.name - handle.close() - try: - sf.write(prompt_wav, waveform.numpy(), sample_rate_hz, format="WAV") - except Exception: - try: - os.unlink(prompt_wav) - except FileNotFoundError: - pass - raise - self._owned_prompt_wavs[state_id] = (prompt_cache_id, prompt_wav) - return prompt_cache_id, prompt_wav - def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor: return torch.zeros((input_ids.numel(), 1), device=input_ids.device, dtype=torch.float32) def compute_logits(self, hidden_states: Any, sampling_metadata: Any = None) -> None: return None + def _materialize_runtime_prompt( + self, + ref_audio: Any, + sample_rate: Any, + ) -> tuple[str, _RuntimePrompt]: + sample_rate_hz = int(_scalar(sample_rate, 0)) + waveform = torch.as_tensor(ref_audio, dtype=torch.float32).reshape(-1).cpu().contiguous() + if sample_rate_hz <= 0: + raise _batch_error("invalid_ref_audio_sample_rate", sample_rate=sample_rate_hz) + if waveform.numel() == 0: + raise _batch_error("empty_ref_audio") + if not bool(torch.isfinite(waveform).all().item()): + raise _batch_error("non_finite_ref_audio") + + digest = sha256() + digest.update(waveform.numpy().tobytes()) + digest.update(str(sample_rate_hz).encode()) + cache_key = digest.hexdigest() + cache_id = f"runtime-ref-{cache_key[:24]}-{sample_rate_hz}" + path = str(Path(self._runtime_prompt_dir.name) / f"minicpmo45_ref_{cache_key[:24]}_{sample_rate_hz}.wav") + entry = self._runtime_prompts.get(cache_key) + if entry is None: + entry = _RuntimePrompt(cache_id=cache_id, path=path, owners=set()) + self._runtime_prompts[cache_key] = entry + prompt_path = Path(entry.path) + if not prompt_path.is_file(): + with tempfile.NamedTemporaryFile( + dir=prompt_path.parent, + prefix=f".{prompt_path.stem}-", + suffix=".wav", + delete=False, + ) as handle: + temporary_path = Path(handle.name) + try: + sf.write( + temporary_path, + waveform.numpy(), + sample_rate_hz, + format="WAV", + ) + os.replace(temporary_path, prompt_path) + finally: + temporary_path.unlink(missing_ok=True) + return cache_key, entry + + def _resolve_prompt( + self, + state_id: str, + info: Mapping[str, Any], + meta: Mapping[str, Any], + previous: _RequestState | None, + ) -> tuple[str, str, str | None]: + codes = info.get("codes") + ref_audio = codes.get("ref") if isinstance(codes, Mapping) else None + if ref_audio is not None: + cache_key, entry = self._materialize_runtime_prompt( + ref_audio, + meta.get("ref_audio_sr"), + ) + return entry.cache_id, entry.path, cache_key + + if previous is not None: + return previous.prompt_cache_id, previous.prompt_wav, self._request_prompt_keys.get(state_id) + + cache_key = self._request_prompt_keys.get(state_id) + entry = self._runtime_prompts.get(cache_key) if cache_key is not None else None + if entry is not None: + return entry.cache_id, entry.path, cache_key + + return ( + str(_scalar(meta.get("prompt_cache_id"), self._default_prompt_id)), + str(_scalar(meta.get("prompt_wav"), self._default_prompt_wav)), + None, + ) + + def _release_request_prompt(self, state_id: str) -> None: + cache_key = self._request_prompt_keys.pop(state_id, None) + entry = self._runtime_prompts.get(cache_key) if cache_key is not None else None + if entry is None: + return + entry.owners.discard(state_id) + if entry.owners: + return + if self.backend is not None: + self.backend.evict_prompt(entry.cache_id, entry.path) + Path(entry.path).unlink(missing_ok=True) + self._runtime_prompts.pop(cache_key, None) + + def _commit_runtime_prompt_owners(self, items: list[_WorkItem]) -> None: + for item in items: + cache_key = item.runtime_prompt_key + if cache_key is None: + continue + previous_key = self._request_prompt_keys.get(item.state_id) + if previous_key != cache_key: + self._release_request_prompt(item.state_id) + entry = self._runtime_prompts.get(cache_key) + if entry is not None: + entry.owners.add(item.state_id) + self._request_prompt_keys[item.state_id] = cache_key + + def _prune_unowned_runtime_prompts(self) -> None: + for cache_key, entry in list(self._runtime_prompts.items()): + if entry.owners: + continue + if self.backend is not None: + self.backend.evict_prompt(entry.cache_id, entry.path) + Path(entry.path).unlink(missing_ok=True) + self._runtime_prompts.pop(cache_key, None) + @staticmethod def _split_segments(input_ids: torch.Tensor, counts: Any) -> list[torch.Tensor]: flat = input_ids.reshape(-1) @@ -263,9 +335,11 @@ class MiniCPMO45Code2Wav(nn.Module): last_chunk=False, tokens=segment.new_empty(0, dtype=torch.long), previous=None, - segment_text="", - duplex_turn_id=None, - duplex_epoch=None, + runtime_prompt_key=None, + duplex_epoch=-1, + duplex_turn_id=-1, + segment_text_utf8=torch.empty(0, dtype=torch.uint8), + tts_is_last_chunk=False, segment_end=False, turn_end=False, has_payload=False, @@ -282,14 +356,15 @@ class MiniCPMO45Code2Wav(nn.Module): chunk_seq=chunk_seq, ) last_chunk = bool(_scalar(meta.get("last_chunk"), False)) + tts_is_last_chunk = bool(_scalar(meta.get("tts_is_last_chunk"), False)) codes = info.get("codes") audio = codes.get("audio") if isinstance(codes, Mapping) else None - reference = codes.get("ref") if isinstance(codes, Mapping) else None tokens = _codec_tensor(audio, segment) - if last_chunk and int(_scalar(meta.get("code_flat_numel"), tokens.numel())) == 0: + if int(_scalar(meta.get("code_flat_numel"), tokens.numel())) == 0: # The generation scheduler reserves one placeholder token for an - # empty terminal chunk. The producer's explicit length is the - # authority, so do not decode that placeholder as codec data. + # empty terminal or segment-boundary chunk. The producer's + # explicit length is the authority, so do not decode that + # placeholder as codec data. tokens = segment.new_empty(0, dtype=torch.long) previous = self._states.get(state_id) if previous is None: @@ -315,7 +390,6 @@ class MiniCPMO45Code2Wav(nn.Module): cache_epoch=cache_epoch, chunk_seq=chunk_seq, ) - self._release_prompt(state_id) previous = None elif chunk_seq != previous.chunk_seq + 1: raise _batch_error( @@ -324,22 +398,12 @@ class MiniCPMO45Code2Wav(nn.Module): expected=previous.chunk_seq + 1, actual=chunk_seq, ) - - explicit_prompt_id = _scalar(meta.get("prompt_cache_id")) - explicit_prompt_wav = _scalar(meta.get("prompt_wav")) - if previous is not None and explicit_prompt_id is None and explicit_prompt_wav is None: - prompt_cache_id = previous.prompt_cache_id - prompt_wav = previous.prompt_wav - elif reference is not None: - prompt_cache_id, prompt_wav = self._write_reference_prompt( - state_id, - reference, - meta.get("ref_audio_sr"), - ) - else: - prompt_cache_id = str(explicit_prompt_id or self._default_prompt_id) - prompt_wav = str(explicit_prompt_wav or self._default_prompt_wav) - + prompt_cache_id, prompt_wav, runtime_prompt_key = self._resolve_prompt( + state_id, + info, + meta, + previous, + ) if previous is not None and prompt_cache_id != previous.prompt_cache_id: raise _batch_error( "prompt_changed_midstream", @@ -354,8 +418,9 @@ class MiniCPMO45Code2Wav(nn.Module): expected=previous.prompt_wav, actual=prompt_wav, ) - duplex_turn_id = _scalar(meta.get("duplex_turn_id")) - duplex_epoch = _scalar(meta.get("duplex_epoch")) + segment_text_utf8 = meta.get("llm_output_text_utf8") + if not isinstance(segment_text_utf8, torch.Tensor): + segment_text_utf8 = torch.empty(0, dtype=torch.uint8) return _WorkItem( output_index=index, state_id=state_id, @@ -367,9 +432,11 @@ class MiniCPMO45Code2Wav(nn.Module): last_chunk=last_chunk, tokens=tokens, previous=previous, - segment_text=str(_scalar(meta.get("native_duplex_segment_text"), "")), - duplex_turn_id=int(duplex_turn_id) if duplex_turn_id is not None else None, - duplex_epoch=int(duplex_epoch) if duplex_epoch is not None else None, + runtime_prompt_key=runtime_prompt_key, + duplex_epoch=int(_scalar(meta.get("duplex_epoch"), -1)), + duplex_turn_id=int(_scalar(meta.get("duplex_turn_id"), -1)), + segment_text_utf8=segment_text_utf8, + tts_is_last_chunk=tts_is_last_chunk, segment_end=bool(_scalar(meta.get("segment_end"), False)), turn_end=bool(_scalar(meta.get("turn_end"), False)), ) @@ -387,6 +454,7 @@ class MiniCPMO45Code2Wav(nn.Module): int(item.tokens.numel()), cache_signature, item.last_chunk, + item.tts_is_last_chunk, item.cache_epoch, ) @@ -451,26 +519,37 @@ class MiniCPMO45Code2Wav(nn.Module): request_ids=len(state_ids), ) items: list[_WorkItem] = [] - for index, (state_id, segment, info) in enumerate( - zip(state_ids, segments, runtime_additional_information, strict=True) - ): - if not isinstance(info, Mapping): - raise _batch_error( - "invalid_runtime_info", - output_index=index, - value_type=type(info).__name__, - ) - items.append(self._parse_item(index, str(state_id), segment, info)) + try: + for index, (state_id, segment, info) in enumerate( + zip(state_ids, segments, runtime_additional_information, strict=True) + ): + if not isinstance(info, Mapping): + raise _batch_error( + "invalid_runtime_info", + output_index=index, + value_type=type(info).__name__, + ) + items.append(self._parse_item(index, str(state_id), segment, info)) + except Exception: + self._prune_unowned_runtime_prompts() + raise state_ids = [item.state_id for item in items] if len(state_ids) != len(set(state_ids)): + self._prune_unowned_runtime_prompts() raise _batch_error("duplicate_request_in_forward", request_ids=state_ids) outputs = [empty for _ in segments] sentinels = [item for item in items if item.last_chunk and item.tokens.numel() == 0] + segment_markers = [ + item for item in items if not item.last_chunk and item.tts_is_last_chunk and item.tokens.numel() == 0 + ] compute_items = [item for item in items if item.tokens.numel() > 0] invalid_empty = [ - item.request_id for item in items if item.has_payload and not item.last_chunk and item.tokens.numel() == 0 + item.request_id + for item in items + if item.has_payload and not item.last_chunk and not item.tts_is_last_chunk and item.tokens.numel() == 0 ] if invalid_empty: + self._prune_unowned_runtime_prompts() raise _batch_error("empty_nonfinal_chunk", request_ids=invalid_empty) buckets: dict[tuple[Any, ...], list[_WorkItem]] = {} @@ -486,6 +565,7 @@ class MiniCPMO45Code2Wav(nn.Module): if len(bucket) < self._min_batch_size ] if undersized: + self._prune_unowned_runtime_prompts() raise _batch_error( "exact_shape_bucket_below_minimum", minimum=self._min_batch_size, @@ -493,6 +573,58 @@ class MiniCPMO45Code2Wav(nn.Module): ) pending: dict[str, _RequestState | None] = {item.state_id: None for item in sentinels} + pending.update( + { + item.state_id: _RequestState( + cache_epoch=item.cache_epoch, + chunk_seq=item.chunk_seq, + prompt_cache_id=item.prompt_cache_id, + prompt_wav=item.prompt_wav, + token2wav=item.previous.token2wav, + ) + for item in segment_markers + if item.previous is not None + } + ) + initial_marker_buckets: dict[tuple[str, str], list[_WorkItem]] = {} + for item in segment_markers: + if item.previous is None: + initial_marker_buckets.setdefault( + (item.prompt_cache_id, item.prompt_wav), + [], + ).append(item) + for bucket in initial_marker_buckets.values(): + try: + features = self.backend.prepare_prompt( + bucket[0].prompt_cache_id, + bucket[0].prompt_wav, + ) + states = self.backend.setup_batch(features, len(bucket)) + except Exception as exc: + self._prune_unowned_runtime_prompts() + if isinstance(exc, RuntimeError) and str(exc).startswith("MiniCPMO45Code2WavBatchError "): + raise + raise _batch_error( + "backend_unsupported_or_failed", + request_ids=[item.request_id for item in bucket], + error_type=type(exc).__name__, + error=str(exc), + ) from exc + if len(states) != len(bucket): + self._prune_unowned_runtime_prompts() + raise _batch_error( + "backend_result_size_mismatch", + expected=len(bucket), + states=len(states), + ) + for item, state in zip(bucket, states, strict=True): + pending[item.state_id] = _RequestState( + cache_epoch=item.cache_epoch, + chunk_seq=item.chunk_seq, + prompt_cache_id=item.prompt_cache_id, + prompt_wav=item.prompt_wav, + token2wav=state, + ) for bucket in buckets.values(): batch_size = len(bucket) try: @@ -512,6 +644,7 @@ class MiniCPMO45Code2Wav(nn.Module): last_chunk=bucket[0].last_chunk, ) except Exception as exc: + self._prune_unowned_runtime_prompts() if isinstance(exc, RuntimeError) and str(exc).startswith("MiniCPMO45Code2WavBatchError "): raise raise _batch_error( @@ -521,6 +654,7 @@ class MiniCPMO45Code2Wav(nn.Module): error=str(exc), ) from exc if len(audios) != batch_size or len(next_states) != batch_size: + self._prune_unowned_runtime_prompts() raise _batch_error( "backend_result_size_mismatch", expected=batch_size, @@ -541,10 +675,10 @@ class MiniCPMO45Code2Wav(nn.Module): ) ) + self._commit_runtime_prompt_owners(items) for request_id, state in pending.items(): if state is None: self._states.pop(request_id, None) - self._release_prompt(request_id) else: self._states[request_id] = state sample_rate_tensor = torch.as_tensor(sample_rate, dtype=torch.int32) @@ -553,24 +687,15 @@ class MiniCPMO45Code2Wav(nn.Module): multimodal_outputs={ "model_outputs": outputs, "sr": [sample_rate_tensor.clone() for _ in outputs], - "meta.llm_output_text_utf8": [ - torch.tensor(list(item.segment_text.encode("utf-8")), dtype=torch.uint8) for item in items - ], - "meta.tts_is_last_chunk": [torch.tensor(item.last_chunk, dtype=torch.bool) for item in items], + # Generation runner wire payloads are flat and tensor-only. + # Dotted metadata keys are unflattened again by the output + # processor before the full-duplex data plane consumes them. + "meta.duplex_epoch": [torch.tensor(item.duplex_epoch, dtype=torch.int32) for item in items], + "meta.duplex_turn_id": [torch.tensor(item.duplex_turn_id, dtype=torch.int32) for item in items], + "meta.llm_output_text_utf8": [item.segment_text_utf8 for item in items], + "meta.tts_is_last_chunk": [torch.tensor(item.tts_is_last_chunk, dtype=torch.bool) for item in items], "meta.segment_end": [torch.tensor(item.segment_end, dtype=torch.bool) for item in items], "meta.turn_end": [torch.tensor(item.turn_end, dtype=torch.bool) for item in items], - "meta.duplex_turn_id": [ - torch.tensor(item.duplex_turn_id, dtype=torch.int64) - if item.duplex_turn_id is not None - else torch.empty(0, dtype=torch.int64) - for item in items - ], - "meta.duplex_epoch": [ - torch.tensor(item.duplex_epoch, dtype=torch.int64) - if item.duplex_epoch is not None - else torch.empty(0, dtype=torch.int64) - for item in items - ], }, ) @@ -578,7 +703,7 @@ class MiniCPMO45Code2Wav(nn.Module): for request_id in finished_req_ids: state_id = str(request_id) self._states.pop(state_id, None) - self._release_prompt(state_id) + self._release_request_prompt(state_id) def make_omni_output(self, model_outputs: Any, **_: Any) -> OmniOutput: if isinstance(model_outputs, OmniOutput): diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py index c9083c261..f2733f3b7 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py @@ -175,7 +175,6 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp token_ids = self._minicpmo45_native_duplex_token_ids() listen_id = int(token_ids.get("listen_token_id", -1)) - tts_bos_id = int(token_ids.get("tts_bos_token_id", -1)) turn_eos_id = int(token_ids.get("turn_eos_token_id", -1)) if listen_id < 0 or listen_id >= logits.shape[-1]: return @@ -199,7 +198,6 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp continue force_listen = payload.get("force_listen") is True is_speech = payload.get("is_speech") - redirect_listen = False segment_key = (row.request_id, row.seq if row.seq is not None else -1) session_key = (row.session_id, row.incarnation) if row.session_id is not None else None if turn_eos_id >= 0 and session_key is not None: @@ -209,8 +207,6 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp ) if is_speech is True: if state is not None: - if not getattr(state, "current_turn_ended", True): - redirect_listen = True with suppress(Exception): state.last_terminator_token = None else: @@ -218,7 +214,7 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp if turn_ended and not pending_speech_context: force_listen = True - if not force_listen and not redirect_listen: + if not force_listen: continue if force_listen and segment_key in force_listen_segments: continue @@ -226,10 +222,6 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp logits[row_idx, :] = float("-inf") logits[row_idx, listen_id] = 0.0 force_listen_segments.add(segment_key) - elif 0 <= tts_bos_id < logits.shape[-1]: - if logits[row_idx, listen_id] > logits[row_idx, tts_bos_id]: - logits[row_idx, tts_bos_id] = logits[row_idx, listen_id] - logits[row_idx, listen_id] = float("-inf") # -------------------- Device utilities -------------------- @staticmethod @@ -730,6 +722,10 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp output_token_ids = getattr(sampling_metadata, "output_token_ids", None) or [] raw_recent_tokens = output_token_ids[row_idx] if row_idx < len(output_token_ids) else [] recent_tokens = [int(token_id) for token_id in raw_recent_tokens if isinstance(token_id, int) and token_id >= 0] + temperature = float(self._sampling_metadata_value(sampling_metadata, "temperature", row_idx, 0.7)) + top_k = int(self._sampling_metadata_value(sampling_metadata, "top_k", row_idx, 100)) + top_p = float(self._sampling_metadata_value(sampling_metadata, "top_p", row_idx, 0.8)) + state = self._minicpmo45_duplex_state_for_row(row_idx) if chunk_eos_id >= 0 and chunk_eos_id < logits.shape[-1]: max_speak_tokens = int( getattr( @@ -765,23 +761,19 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp if valid_forbidden: logits[:, valid_forbidden] = float("-inf") - special_ids = self._minicpmo45_native_special_token_ids(token_ids) - state = self._minicpmo45_duplex_state_for_row(row_idx) - generated_text_tokens = getattr(state, "generated_text_tokens", None) - repetition_tokens = generated_text_tokens if generated_text_tokens else recent_tokens + generated_tokens = getattr(state, "generated_tokens", None) + repetition_tokens = generated_tokens if generated_tokens else recent_tokens repetition_penalty = 1.05 if repetition_penalty != 1.0 and repetition_tokens: history_size = MiniCPMO45DuplexPolicy.REPETITION_HISTORY_SIZE for token_id in set(repetition_tokens[-history_size:]): - if token_id in special_ids or token_id < 0 or token_id >= logits.shape[-1]: + if token_id < 0 or token_id >= logits.shape[-1]: continue logits[0, token_id] /= repetition_penalty - temperature = float(self._sampling_metadata_value(sampling_metadata, "temperature", row_idx, 0.7)) - top_k = int(self._sampling_metadata_value(sampling_metadata, "top_k", row_idx, 100)) - top_p = float(self._sampling_metadata_value(sampling_metadata, "top_p", row_idx, 0.8)) if getattr(sampling_metadata, "all_greedy", False) or temperature <= 0: sampled = int(torch.argmax(logits, dim=-1).item()) + self._record_minicpmo45_duplex_generation_token(row_idx, sampled) sampled = self._maybe_cut_minicpmo45_native_duplex_text_chunk( sampled, recent_tokens, @@ -793,6 +785,7 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp logits = self._top_k_top_p_filter(logits, top_k=top_k, top_p=top_p) probs = F.softmax(logits, dim=-1) sampled = int(torch.multinomial(probs, num_samples=1, generator=generator).item()) + self._record_minicpmo45_duplex_generation_token(row_idx, sampled) sampled = self._maybe_cut_minicpmo45_native_duplex_text_chunk( sampled, recent_tokens, @@ -901,6 +894,28 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp return int(tts_bos_id) return int(sampled) + def _record_minicpmo45_duplex_generation_token(self, row_idx: int, sampled: int) -> None: + """Track tokens returned by the model-policy decoder. + + The released StreamDecoder is constructed without special_token_ids, so + every normally decoded token participates in repetition penalty. Forced + listen bypasses that decoder, while chunk_eos boundary decisions return + before this method is called. + """ + state = self._minicpmo45_duplex_state_for_row(row_idx) + if state is None: + return + payload = self._minicpmo45_duplex_payload_for_row(row_idx) + if isinstance(payload, dict) and payload.get("force_listen") is True: + return + generated_tokens = getattr(state, "generated_tokens", None) + if not isinstance(generated_tokens, list): + generated_tokens = [] + state.generated_tokens = generated_tokens + generated_tokens.append(int(sampled)) + history_size = MiniCPMO45DuplexPolicy.REPETITION_HISTORY_SIZE + del generated_tokens[:-history_size] + def _record_minicpmo45_duplex_terminator(self, row_idx: int, sampled: int, token_ids: dict[str, int]) -> None: """Remember sampled unit state for the next append. @@ -928,15 +943,6 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp with suppress(Exception): state.pending_speech_response_open = False return - special_ids = self._minicpmo45_native_special_token_ids(token_ids) - if sampled not in special_ids: - generated_text_tokens = getattr(state, "generated_text_tokens", None) - if not isinstance(generated_text_tokens, list): - generated_text_tokens = [] - state.generated_text_tokens = generated_text_tokens - generated_text_tokens.append(int(sampled)) - history_size = MiniCPMO45DuplexPolicy.REPETITION_HISTORY_SIZE - del generated_text_tokens[:-history_size] if ( sampled == tts_bos_id and getattr(state, "current_turn_ended", True) diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py index 1ff37a88e..02b0bf91f 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py @@ -43,6 +43,7 @@ _CODEC_TOP_K = 25 _CODEC_TOP_P = 0.85 _CODEC_REPETITION_PENALTY = 1.05 _CODEC_MIN_TOKENS = 50 +_DUPLEX_CODEC_TOKENS_PER_CHUNK = 26 def _max_audio_tokens(condition_tokens: int) -> int: @@ -61,12 +62,7 @@ def _max_audio_tokens(condition_tokens: int) -> int: def _restore_weight_norm_weight(weight_g: torch.Tensor, weight_v: torch.Tensor) -> torch.Tensor: """Materialize ``weight_norm(..., dim=0)`` checkpoint parameters.""" - row_norm = torch.linalg.vector_norm( - weight_v.float(), - dim=1, - keepdim=True, - ).clamp_min(torch.finfo(torch.float32).tiny) - return weight_v * (weight_g.to(dtype=weight_v.dtype) / row_norm.to(dtype=weight_v.dtype)) + return torch._weight_norm(weight_v, weight_g, dim=0) def _apply_repetition_penalty( @@ -218,6 +214,8 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): self, tts_token_ids: torch.Tensor, tts_hidden_states: torch.Tensor, + *, + native_duplex: bool = False, ) -> torch.Tensor: if tts_token_ids.numel() == 0 or tts_hidden_states.numel() == 0: # The thinker can legally emit an empty speech segment (<|tts_bos|> @@ -239,7 +237,12 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): hidden_embeds = self.projector_semantic(hidden) if self._normalize: hidden_embeds = F.normalize(hidden_embeds, p=2, dim=-1) - return torch.cat([text_embeds + hidden_embeds, self._boundary_embeddings()], dim=0) + audio_bos = self.emb_text(torch.tensor([self._tts_bos_id], device=device, dtype=torch.long)) + condition = text_embeds + hidden_embeds + if native_duplex: + # Match MiniCPMTTS.generate_chunk's streaming condition. + return torch.cat([condition, audio_bos], dim=0) + return torch.cat([condition, self._boundary_embeddings()], dim=0) def preprocess( self, @@ -281,18 +284,22 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): "MiniCPM-o Talker received an empty condition (request %s); this request produces no audio.", info_dict.get("request_id"), ) - full_embeds = self._build_condition_embeddings(token_ids, hidden_states) + native_duplex = bool(info_dict.get("native_duplex", False)) + full_embeds = self._build_condition_embeddings( + token_ids, + hidden_states, + native_duplex=native_duplex, + ) offset = int(info_dict.get("_omni_num_computed_tokens", 0)) - if offset == 0 and span_len > full_embeds.shape[0]: - # Async-chunk prewarms Stage 1 with placeholder token IDs. Two - # Thinker handoffs can arrive before the first one executes, - # so the scheduler prompt may already include the newer - # placeholder prefix while additional_information still - # carries the preceding condition. Materialize that prefix as - # the embeddings of its actual zero token IDs and keep the - # complete Talker condition tail-aligned (audio BOS remains - # the final prefill token). - prefix_len = span_len - full_embeds.shape[0] + request_id = str(info_dict.get("request_id", "0")) + meta = info_dict.get("meta") + # The handoff rebuilds only the tail-aligned Talker condition. + # Materialize zero-token embeddings for any scheduler prompt + # prefix so chunked prefill can slice from a non-zero offset. + prompt_len = info_dict.get("_omni_prompt_len") + target_len = int(prompt_len) if prompt_len is not None else offset + span_len + prefix_len = target_len - full_embeds.shape[0] + if prefix_len > 0: placeholder_ids = torch.zeros( prefix_len, dtype=torch.long, @@ -308,13 +315,21 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): f"tts_ids={token_ids.shape[0]} tts_hidden={hidden_states.shape[0]} " f"prompt_len={info_dict.get('_omni_prompt_len')}" ) - max_tokens = _max_audio_tokens(int(token_ids.numel())) + duplex_boundary = isinstance(meta, dict) and ( + bool(meta.get("turn_start", False)) or bool(meta.get("turn_end", False)) + ) + if native_duplex: + max_tokens = _DUPLEX_CODEC_TOKENS_PER_CHUNK + min_tokens = 0 if duplex_boundary else _DUPLEX_CODEC_TOKENS_PER_CHUNK + else: + max_tokens = _max_audio_tokens(int(token_ids.numel())) + min_tokens = self._codec_min_tokens state = { "step": 0, "max_tokens": max_tokens, + "min_tokens": min_tokens, "finished": empty_condition, } - request_id = str(info_dict.get("request_id", "0")) request_states = getattr(self, "_request_audio_states", None) if request_states is None: request_states = {} @@ -370,7 +385,12 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): penalty=self._codec_repetition_penalty, window_size=_REPETITION_WINDOW, ) - if step < self._codec_min_tokens: + request_states = getattr(self, "_request_audio_states", {}) + state = request_states.get(request_id) + min_tokens = ( + int(state.get("min_tokens", self._codec_min_tokens)) if isinstance(state, dict) else self._codec_min_tokens + ) + if step < min_tokens: logits[..., eos_id] = float("-inf") logits = _apply_top_k_top_p( logits, @@ -404,12 +424,61 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): raise RuntimeError( f"MiniCPM-o continuous Talker received {len(sample_eligible)} sampling flags for {len(infos)} requests" ) + emit_duplex_metadata = any(isinstance(info, dict) and info.get("native_duplex") is True for info in infos) stop_rows: list[torch.Tensor] = [] codec_deltas: list[torch.Tensor] = [] terminal_flags: list[torch.Tensor] = [] + native_duplex_flags: list[torch.Tensor] = [] + duplex_epochs: list[torch.Tensor] = [] + duplex_turn_ids: list[torch.Tensor] = [] + segment_texts_utf8: list[torch.Tensor] = [] + turn_end_flags: list[torch.Tensor] = [] empty_delta = hidden.new_empty((0, 1), dtype=torch.long) for index, info in enumerate(infos): + info_dict = info if isinstance(info, dict) else {} + native_duplex = info_dict.get("native_duplex") is True + if emit_duplex_metadata: + duplex_info = info_dict.get("duplex") + if not isinstance(duplex_info, dict): + duplex_info = {} + epoch = duplex_info.get("epoch", -1) + turn_id = duplex_info.get("turn_id", -1) + if native_duplex and not all( + isinstance(value, int) and not isinstance(value, bool) and value >= 0 for value in (epoch, turn_id) + ): + raise RuntimeError( + "MiniCPM-o native duplex Talker requires non-negative integer " + f"epoch and turn_id, got epoch={epoch!r}, turn_id={turn_id!r}" + ) + meta_info = info_dict.get("meta") + if not isinstance(meta_info, dict): + meta_info = {} + segment_text = meta_info.get("native_duplex_segment_text", "") if native_duplex else "" + if not isinstance(segment_text, str): + segment_text = "" + turn_eos_id = meta_info.get("turn_eos_token_id") + ids_info = info_dict.get("ids") + tts_ids = ids_info.get("tts") if native_duplex and isinstance(ids_info, dict) else None + if isinstance(tts_ids, torch.Tensor): + contains_turn_eos = isinstance(turn_eos_id, int) and bool( + torch.any(tts_ids.reshape(-1) == turn_eos_id).item() + ) + elif isinstance(tts_ids, (list, tuple)): + contains_turn_eos = isinstance(turn_eos_id, int) and turn_eos_id in tts_ids + else: + contains_turn_eos = False + native_duplex_flags.append(torch.tensor(native_duplex, dtype=torch.bool)) + duplex_epochs.append(torch.tensor(epoch if isinstance(epoch, int) else -1, dtype=torch.long)) + duplex_turn_ids.append(torch.tensor(turn_id if isinstance(turn_id, int) else -1, dtype=torch.long)) + segment_texts_utf8.append( + torch.tensor( + list(segment_text.encode("utf-8")), + dtype=torch.uint8, + ) + ) + turn_end_flags.append(torch.tensor(native_duplex and contains_turn_eos, dtype=torch.bool)) + if not isinstance(info, dict): stop_rows.append(hidden.new_tensor([0.0, float("-inf")])) codec_deltas.append(empty_delta) @@ -453,12 +522,15 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): codes = codes.to(device=hidden.device, dtype=torch.long).reshape(-1) step = int(state.get("step", 0)) sampled = self._sample_audio_code(hidden[end - 1 : end], codes, request_id, step) - is_eos = int(sampled.item()) == self._num_audio_tokens - 1 + sampled_id = int(sampled.item()) + is_eos = sampled_id == self._num_audio_tokens - 1 state["step"] = int(state.get("step", 0)) + 1 reached_limit = int(state["step"]) >= int(state.get("max_tokens", 2048)) finished = is_eos or reached_limit state["finished"] = finished - if not is_eos: + # MiniCPMTTS.generate_chunk consumes the boundary sample but + # returns only codes that were fed into the retained KV state. + if not is_eos and not reached_limit: codes = torch.cat([codes[-(_REPETITION_WINDOW - 1) :], sampled.reshape(1)]) delta = sampled.reshape(1, 1) else: @@ -476,9 +548,20 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): self._batch_stop_logits = torch.stack(stop_rows, dim=0) if stop_rows else hidden.new_empty((0, 2)) # Lists are deliberate: the runner routes element i to request i, # preserving compaction alignment while emitting only this step's code. + meta_outputs = {"finished": terminal_flags} + if emit_duplex_metadata: + meta_outputs.update( + { + "native_duplex": native_duplex_flags, + "duplex_epoch": duplex_epochs, + "duplex_turn_id": duplex_turn_ids, + "llm_output_text_utf8": segment_texts_utf8, + "turn_end": turn_end_flags, + } + ) multimodal_outputs: dict[str, Any] = { "codes": {"audio": codec_deltas}, - "meta": {"finished": terminal_flags}, + "meta": meta_outputs, } return OmniOutput( text_hidden_states=hidden, diff --git a/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py b/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py index 950d5434d..d12a3b382 100644 --- a/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py +++ b/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py @@ -21,6 +21,7 @@ logger = logging.getLogger(__name__) _MINICPMO45_ASYNC_STATE = "_minicpmo45_async_codec_state" _MINICPMO45_STREAM_RECORD = "_minicpmo45_async_stream_record" _MINICPMO45_SILENCE_CODE = 4218 +_MINICPMO45_MIN_STREAM_BODY_FRAMES = 5 class _MiniCPMO45MetaStruct(MetaStruct): @@ -28,6 +29,7 @@ class _MiniCPMO45MetaStruct(MetaStruct): ref_audio_sr: int | None = None native_duplex_segment_text: str | None = None + llm_output_text_utf8: torch.Tensor | None = None duplex_turn_id: int | None = None duplex_epoch: int | None = None segment_end: bool | None = None @@ -212,6 +214,16 @@ def tts2code2wav_async_chunk( internal_id = getattr(request, "request_id", None) request_id = str(external_id if external_id is not None else internal_id) internal_id = str(internal_id if internal_id is not None else request_id) + output_meta = multimodal_output.get("meta") if isinstance(multimodal_output, Mapping) else None + output_meta = output_meta if isinstance(output_meta, Mapping) else {} + native_duplex = bool(_coerce_int(output_meta.get("native_duplex"))) + duplex_epoch = _coerce_int(output_meta.get("duplex_epoch")) + duplex_turn_id = _coerce_int(output_meta.get("duplex_turn_id")) + segment_text_utf8 = output_meta.get("llm_output_text_utf8") + if not isinstance(segment_text_utf8, torch.Tensor): + segment_text_utf8 = None + turn_end = bool(_coerce_int(output_meta.get("turn_end"))) + duplex_turn_key = (duplex_epoch, duplex_turn_id) request_payload = getattr(transfer_manager, "request_payload", None) if request_payload is None: @@ -227,7 +239,9 @@ def tts2code2wav_async_chunk( record = { "internal_id": internal_id, "cache_epoch": 0, + "chunk_seq": 0, "retired_internal_ids": set(), + "last_terminal_turn": None, } container[_MINICPMO45_STREAM_RECORD] = record elif internal_id in record["retired_internal_ids"]: @@ -236,6 +250,8 @@ def tts2code2wav_async_chunk( record["retired_internal_ids"].add(record["internal_id"]) record["internal_id"] = internal_id record["cache_epoch"] = int(record["cache_epoch"]) + 1 + record["chunk_seq"] = 0 + record["last_terminal_turn"] = None _drop_codec_state(transfer_manager, request_id) if _is_aborted(request): @@ -243,44 +259,46 @@ def tts2code2wav_async_chunk( _drop_codec_state(transfer_manager, request_id) return None + if native_duplex and turn_end and record.get("last_terminal_turn") == duplex_turn_key: + return None + state = container.get(_MINICPMO45_ASYNC_STATE) if not isinstance(state, dict): - request_info = getattr(request, "additional_information", None) - if not isinstance(request_info, Mapping): - request_info = {} - codes_info = request_info.get("codes") - if not isinstance(codes_info, Mapping): - codes_info = {} - meta_info = request_info.get("meta") - if not isinstance(meta_info, Mapping): - meta_info = {} - duplex_info = request_info.get("duplex") - if not isinstance(duplex_info, Mapping): - duplex_info = {} state = { "internal_id": internal_id, "pending": [], + "pending_text_utf8": [], + "segment_text_recorded": False, "left_context": [], "codec_end": 0, - "ref_audio": codes_info.get("ref"), - "ref_audio_sr": meta_info.get("ref_audio_sr"), - "segment_text": meta_info.get("native_duplex_segment_text"), - "segment_end": bool(meta_info.get("segment_end", False)), - "turn_end": bool(meta_info.get("turn_end", False)), - "duplex_turn_id": duplex_info.get("model_turn_id", duplex_info.get("turn_id")), - "duplex_epoch": duplex_info.get("epoch"), } container[_MINICPMO45_ASYNC_STATE] = state pending = state["pending"] pending.extend(_extract_codec_delta(multimodal_output, request_id)) + pending_text_utf8 = state.setdefault("pending_text_utf8", []) + current_text_utf8 = ( + segment_text_utf8.detach().to(device="cpu", dtype=torch.uint8).reshape(-1).tolist() + if isinstance(segment_text_utf8, torch.Tensor) + else [] + ) + if native_duplex and current_text_utf8 and not state.get("segment_text_recorded", False): + # Talker repeats the unit text on every codec step. Queue it once for + # the first Code2Wav payload that can carry audio for this unit. + pending_text_utf8.extend(current_text_utf8) + state["segment_text_recorded"] = True request_finished = getattr(request, "is_finished", None) finished = bool(is_finished or (callable(request_finished) and request_finished())) chunk_frames, left_context_frames = _codec_config(transfer_manager) - if not finished and len(pending) < chunk_frames: + flush_pending = finished + last_chunk = bool(flush_pending and (not native_duplex or turn_end)) + if not flush_pending and len(pending) < chunk_frames: return None - new_token_count = len(pending) if finished else chunk_frames + hold_short_unit = ( + native_duplex and flush_pending and not last_chunk and 0 < len(pending) < _MINICPMO45_MIN_STREAM_BODY_FRAMES + ) + new_token_count = 0 if hold_short_unit else (len(pending) if flush_pending else chunk_frames) new_codes = pending[:new_token_count] del pending[:new_token_count] codec_start = int(state["codec_end"]) @@ -292,53 +310,79 @@ def tts2code2wav_async_chunk( else: context = list(state["left_context"]) output_codes = [*context, *new_codes] - history = [*state["left_context"], *new_codes] + history = output_codes state["left_context"] = history[-left_context_frames:] if left_context_frames else [] - elif finished and codec_start > 0 and state["left_context"]: + elif last_chunk and codec_start > 0 and state["left_context"]: context = list(state["left_context"]) output_codes = context else: context = [] output_codes = [] state["codec_end"] = codec_end + code_flat_numel = len(output_codes) + if native_duplex and code_flat_numel > 0: + segment_text_utf8 = torch.tensor(pending_text_utf8, dtype=torch.uint8) + pending_text_utf8.clear() + if native_duplex and finished: + state["segment_text_recorded"] = False + if flush_pending and not last_chunk and code_flat_numel == 0: + # Keep the generic generation connector model-agnostic: a real token + # makes this control-only TTS boundary schedulable, while the explicit + # zero length tells Code2Wav to discard the placeholder. + output_codes = [0] - last_chunk = bool(finished and not pending) - if last_chunk: + if last_chunk and not native_duplex: record["retired_internal_ids"].add(internal_id) _drop_codec_state(transfer_manager, request_id) - chunk_seq = int(getattr(transfer_manager, "put_req_chunk", {}).get(request_id, 0)) + chunk_seq = int(record["chunk_seq"]) + record["chunk_seq"] = chunk_seq + 1 + ref_audio = None + ref_audio_sr = None + if int(record["cache_epoch"]) == 0 and chunk_seq == 0: + request_info = getattr(request, "additional_information", None) + if isinstance(request_info, Mapping): + codes_info = request_info.get("codes") + meta_info = request_info.get("meta") + raw_ref_audio = codes_info.get("ref") if isinstance(codes_info, Mapping) else None + raw_ref_audio_sr = meta_info.get("ref_audio_sr") if isinstance(meta_info, Mapping) else None + ref_audio_sr = _coerce_int(raw_ref_audio_sr) + if raw_ref_audio is not None: + ref_audio = torch.as_tensor(raw_ref_audio, dtype=torch.float32).reshape(-1).cpu() finished_tensor = torch.tensor(last_chunk, dtype=torch.bool) - ref_audio = state.get("ref_audio") if codec_start == 0 else None - return OmniPayloadStruct( + payload = OmniPayloadStruct( codes=CodesStruct( audio=torch.tensor(output_codes, dtype=torch.long), - ref=torch.as_tensor(ref_audio, dtype=torch.float32).reshape(-1) if ref_audio is not None else None, + ref=ref_audio, ), meta=_MiniCPMO45MetaStruct( request_id=request_id, chunk_seq=chunk_seq, cache_epoch=int(record["cache_epoch"]), - code_flat_numel=len(output_codes), + code_flat_numel=code_flat_numel, codec_chunk_frames=new_token_count, codec_left_context_frames=len(context), left_context_size=len(context), last_chunk=last_chunk, stream_finished=finished_tensor, finished=finished_tensor, + is_segment_finished=finished_tensor, req_id=[request_id], - ref_audio_sr=_coerce_int(state.get("ref_audio_sr")), - native_duplex_segment_text=( - str(state["segment_text"]) if isinstance(state.get("segment_text"), str) else None - ), - duplex_turn_id=_coerce_int(state.get("duplex_turn_id")), - duplex_epoch=_coerce_int(state.get("duplex_epoch")), - segment_end=bool(state.get("segment_end", False)), - turn_end=bool(state.get("turn_end", False)), - tts_is_last_chunk=last_chunk, + duplex_epoch=duplex_epoch, + duplex_turn_id=duplex_turn_id, + llm_output_text_utf8=segment_text_utf8, + tts_is_last_chunk=flush_pending, + turn_end=turn_end and last_chunk, + ref_audio_sr=ref_audio_sr, ), request_id=request_id, ) + if last_chunk and native_duplex: + record["last_terminal_turn"] = duplex_turn_key + record["cache_epoch"] = int(record["cache_epoch"]) + 1 + record["chunk_seq"] = 0 + _drop_codec_state(transfer_manager, request_id) + return payload def tts2code2wav_full_payload( @@ -496,13 +540,33 @@ def _native_tts_boundary_token_ids(special_token_ids): } +def _decode_native_duplex_token_ids( + token_ids: list[int], + streaming_context, + *, + request_id: str, +) -> str | None: + decode_token_ids = getattr(streaming_context, "source_token_decoder", None) + if not isinstance(decode_token_ids, Callable): + return None + decode_ids = [int(token_id) for token_id in token_ids] + try: + try: + return str(decode_token_ids(decode_ids, skip_special_tokens=True)) + except TypeError: + return str(decode_token_ids(decode_ids)) + except Exception: + logger.exception("Failed to decode MiniCPM-o duplex token delta for request_id=%s", request_id) + return "" + + def _native_duplex_segment_output_ids( output_ids: list[int], output_text: str, streaming_context, *, request_id: str, -) -> tuple[list[int], str]: +) -> tuple[list[int], str, bool]: """Slice the cumulative thinker output down to the current segment. Tracks how many output tokens were already handed to the talker in the @@ -511,7 +575,7 @@ def _native_duplex_segment_output_ids( """ bridge_states = getattr(streaming_context, "bridge_states", None) if not isinstance(bridge_states, dict): - return output_ids, output_text + return output_ids, output_text, True state = bridge_states.setdefault("minicpmo45_tts_handoff", {}) duplex_state = bridge_states.get("duplex") turn_id = duplex_state.get("model_turn_id", duplex_state.get("turn_id")) if isinstance(duplex_state, dict) else None @@ -521,17 +585,12 @@ def _native_duplex_segment_output_ids( state["request_id"] = request_id state["sent_output_len"] = 0 state["sent_output_ids"] = [] - state["acc_tts_ids"] = [] - state["acc_tts_hidden"] = [] + previous_turn_id = state.get("turn_id") sent_len = state.get("sent_output_len", 0) prev_output_ids = state.get("sent_output_ids", []) - prev_turn_id = state.get("turn_id") if not isinstance(sent_len, int) or sent_len < 0 or sent_len > len(output_ids): - # Shrunken cumulative output = epoch reset after barge-in: the talker - # condition history is stale too. + # Shrunken cumulative output = epoch reset after barge-in. sent_len = 0 - state["acc_tts_ids"] = [] - state["acc_tts_hidden"] = [] elif sent_len and isinstance(prev_output_ids, list): prev_prefix = prev_output_ids[:sent_len] current_prefix = output_ids[:sent_len] @@ -540,27 +599,15 @@ def _native_duplex_segment_output_ids( # others keep returning cumulative ids. Detect restart from the # token prefix instead of clearing the cursor at turn_eos. sent_len = 0 - state["acc_tts_ids"] = [] - state["acc_tts_hidden"] = [] - if isinstance(prev_turn_id, int) and isinstance(turn_id, int) and prev_turn_id != turn_id: - # The thinker output list is cumulative across clean turns, so the - # output cursor must stay put. The talker condition is per assistant - # turn, though; carrying it across turn_id boundaries makes stage 1 - # replay the previous turn after its consumed cursor has been reset. - state["acc_tts_ids"] = [] - state["acc_tts_hidden"] = [] + turn_start = sent_len == 0 or previous_turn_id != turn_id segment_ids = output_ids[sent_len:] - decode_token_ids = getattr(streaming_context, "source_token_decoder", None) - if isinstance(decode_token_ids, Callable): - decode_ids = [int(token_id) for token_id in segment_ids] - try: - try: - segment_text = str(decode_token_ids(decode_ids, skip_special_tokens=True)) - except TypeError: - segment_text = str(decode_token_ids(decode_ids)) - except Exception: - logger.exception("Failed to decode MiniCPM-o duplex token delta for request_id=%s", request_id) - segment_text = "" + decoded_segment_text = _decode_native_duplex_token_ids( + segment_ids, + streaming_context, + request_id=request_id, + ) + if decoded_segment_text is not None: + segment_text = decoded_segment_text elif sent_len == 0: # Non-orchestrator unit tests and legacy callers may not install a # decoder. Never slice cumulative text with a stale character cursor. @@ -575,22 +622,7 @@ def _native_duplex_segment_output_ids( state["sent_output_len"] = len(output_ids) state["sent_output_ids"] = list(output_ids) state["turn_id"] = turn_id - return segment_ids, segment_text - - -def _reset_native_tts_handoff(streaming_context) -> None: - bridge_states = getattr(streaming_context, "bridge_states", None) - if not isinstance(bridge_states, dict): - return - state = bridge_states.get("minicpmo45_tts_handoff") - if not isinstance(state, dict): - return - # Keep the output cursor: the thinker can keep reporting cumulative - # output after turn_eos on the same resumable request. If a runtime really - # restarts token ids, _native_duplex_segment_output_ids detects the prefix - # mismatch and resets the slice cursor there. - state["acc_tts_ids"] = [] - state["acc_tts_hidden"] = [] + return segment_ids, segment_text, turn_start def _native_duplex_data_plane_metadata(streaming_context) -> dict[str, object] | None: @@ -623,32 +655,6 @@ def _native_duplex_data_plane_metadata(streaming_context) -> dict[str, object] | return metadata -def _accumulate_native_tts_handoff(streaming_context, new_ids, new_hidden): - """Hand the talker the FULL accumulated condition on every handoff. - - The runner's streaming buffer update is not merge-safe for a resumable - stage-1 request: in-place updates merge sub-keys, but a resume prefill - REPLACES the buffer, silently dropping every earlier segment's tts - tokens/hiddens (observed losing alternating reply segments — the talker - vocalized text it never saw between islands it did). Accumulate here so - the latest handoff always carries the complete history and downstream - replace semantics are lossless; the talker consumes by cursor. - """ - bridge_states = getattr(streaming_context, "bridge_states", None) - if not isinstance(bridge_states, dict): - return new_ids, new_hidden - state = bridge_states.setdefault("minicpmo45_tts_handoff", {}) - acc_ids = state.setdefault("acc_tts_ids", []) - acc_hidden = state.setdefault("acc_tts_hidden", []) - if new_ids: - acc_ids.extend(int(t) for t in new_ids) - if new_hidden: - acc_hidden.extend(new_hidden) - if not acc_ids: - return None, None - return list(acc_ids), list(acc_hidden) - - def _build_tts_scheduler_prompt_token_ids( tts_token_ids: torch.Tensor | None, llm_output_ids: list[int], @@ -723,6 +729,7 @@ def llm2tts( # buffer overflows. llm_output_ids = list(llm_output_ids) thinker_text = getattr(output, "text", "") or "" + native_turn_start = False if _has_native_duplex_prompt_metadata(mm_output): # The thinker's resumable duplex request reports cumulative # output ids/text, but earlier segments are already folded into @@ -732,7 +739,7 @@ def llm2tts( # alignment holds, the talker prompt grows linearly with new # tokens instead of quadratically, and downstream transcripts # carry per-unit deltas instead of re-sending the whole reply. - llm_output_ids, thinker_text = _native_duplex_segment_output_ids( + llm_output_ids, thinker_text, native_turn_start = _native_duplex_segment_output_ids( llm_output_ids, thinker_text, _streaming_context, @@ -875,6 +882,18 @@ def llm2tts( .to(torch.float32) .contiguous() ) + handoff_ids = _coerce_token_id_list(tts_token_ids_slice) if tts_token_ids_slice is not None else None + if is_native_duplex_handoff and handoff_ids: + handoff_text = _decode_native_duplex_token_ids( + handoff_ids, + _streaming_context, + request_id=str(llm_output.request_id), + ) + if handoff_text is not None: + # Match the released streaming_generate contract: transcript + # text comes from total_ids_in_unit, the same slice that + # conditions the Talker. + thinker_text = handoff_text model_intermediate_buffer = build_duplex_intermediate_buffer( request_id=str(llm_output.request_id), prompt_token_ids=prompt_token_ids, @@ -900,6 +919,7 @@ def llm2tts( # The talker detects turn end from <|turn_eos|> inside the # handed condition (official conditions on its embedding). meta["turn_eos_token_id"] = int(turn_eos_id) + meta["turn_start"] = native_turn_start if native_segment_end: meta["segment_end"] = True req_mm_data = multi_modal_data.get(llm_output.request_id) @@ -911,30 +931,27 @@ def llm2tts( if ref_audio is not None: ref_waveform, ref_sr = ref_audio set_ref_audio(model_intermediate_buffer, _to_transport_list(ref_waveform), ref_sr) - handoff_ids = _coerce_token_id_list(tts_token_ids_slice) if tts_token_ids_slice is not None else None handoff_hidden = _to_transport_list(tts_hidden_slice) if tts_hidden_slice is not None else None native_turn_end_handoff = False if is_native_duplex_handoff: turn_eos_id = special_token_ids.get("turn_eos_token_id") native_turn_end_handoff = turn_eos_id is not None and handoff_ids is not None and turn_eos_id in handoff_ids - handoff_ids, handoff_hidden = _accumulate_native_tts_handoff( - _streaming_context, - handoff_ids, - handoff_hidden, - ) if not handoff_ids: continue set_tts_handoff(model_intermediate_buffer, handoff_ids, handoff_hidden) if native_turn_end_handoff: model_intermediate_buffer.setdefault("meta", {})["turn_end"] = True - _reset_native_tts_handoff(_streaming_context) if handoff_ids is not None and handoff_hidden is not None: - condition_length = max(len(handoff_ids), len(handoff_hidden)) + 2 + condition_suffix_length = 1 if is_native_duplex_handoff else 2 + condition_length = max(len(handoff_ids), len(handoff_hidden)) + condition_suffix_length scheduler_prompt_token_ids = [0] * condition_length handoff_meta = model_intermediate_buffer.setdefault("meta", {}) - handoff_meta["replace_streaming_prompt"] = True handoff_meta["next_stage_prompt_len"] = condition_length + # Native duplex resumes one Talker request within a turn, but a new + # assistant turn must discard the previous turn's prompt and KV. + if not is_native_duplex_handoff or native_turn_start: + handoff_meta["replace_streaming_prompt"] = True else: scheduler_prompt_token_ids = _build_tts_scheduler_prompt_token_ids( tts_token_ids_slice, diff --git a/vllm_omni/model_executor/stage_input_processors/qwen3_omni.py b/vllm_omni/model_executor/stage_input_processors/qwen3_omni.py index edde6972d..7e28b94c3 100644 --- a/vllm_omni/model_executor/stage_input_processors/qwen3_omni.py +++ b/vllm_omni/model_executor/stage_input_processors/qwen3_omni.py @@ -685,19 +685,32 @@ def talker2code2wav_async_chunk( """ Multimodal output version. """ - if not isinstance(multimodal_output, Mapping): - return None - talker_codes = multimodal_output.get("codes", {}) - if not isinstance(talker_codes, dict): - return None - code_predictor_codes = talker_codes.get("audio") - if code_predictor_codes is None: - return None + request_id = request.external_req_id + code_predictor_codes = None + if isinstance(multimodal_output, Mapping): + talker_codes = multimodal_output.get("codes", {}) + if isinstance(talker_codes, dict): + code_predictor_codes = talker_codes.get("audio") - if code_predictor_codes.numel() == 0: - return None + sampling_params = getattr(request, "sampling_params", None) + stop_token_ids = set(getattr(sampling_params, "stop_token_ids", None) or []) + stop_token_id = getattr(sampling_params, "stop_token_id", None) + if stop_token_id is not None: + stop_token_ids.add(stop_token_id) - if not code_predictor_codes.any(): + append_codes = ( + isinstance(code_predictor_codes, torch.Tensor) + and code_predictor_codes.numel() > 0 + and bool(code_predictor_codes.any()) + ) + if append_codes: + first_codebook = int(code_predictor_codes[0, 0].item()) + if first_codebook in stop_token_ids: + logger.debug("skip stop-token codec frame: first_codebook=%s", first_codebook) + append_codes = False + if append_codes: + transfer_manager.code_prompt_token_ids[request_id].append(code_predictor_codes) + elif not is_finished: return None connector = getattr(transfer_manager, "connector", None) @@ -707,20 +720,10 @@ def talker2code2wav_async_chunk( left_context_size_config = int(cfg.get("codec_left_context_frames", 25)) configured_initial_chunk_size = int(cfg.get("initial_codec_chunk_frames") or 0) - sampling_params = getattr(request, "sampling_params", None) - stop_token_ids = set(getattr(sampling_params, "stop_token_ids", None) or []) - stop_token_id = getattr(sampling_params, "stop_token_id", None) - if stop_token_id is not None: - stop_token_ids.add(stop_token_id) - first_codebook = int(code_predictor_codes[0, 0].item()) - if first_codebook in stop_token_ids: - logger.debug("skip stop-token codec frame: first_codebook=%s", first_codebook) - return None - - request_id = request.external_req_id chunk_id = transfer_manager.put_req_chunk[request_id] - transfer_manager.code_prompt_token_ids[request_id].append(code_predictor_codes) length = len(transfer_manager.code_prompt_token_ids[request_id]) + if length <= 0: + return None if configured_initial_chunk_size > 0: if chunk_id == 0: @@ -732,6 +735,9 @@ def talker2code2wav_async_chunk( if chunk_length != 0 and not is_finished: return None + if is_finished and not append_codes and chunk_length == 0: + return None + context_length = chunk_length if chunk_length != 0 else chunk_size_config # ensure left context does not exceed available length if configured_initial_chunk_size > 0 and chunk_id == 1: diff --git a/vllm_omni/outputs/multimodal_accumulation.py b/vllm_omni/outputs/multimodal_accumulation.py index 08b2551bc..715f6366c 100644 --- a/vllm_omni/outputs/multimodal_accumulation.py +++ b/vllm_omni/outputs/multimodal_accumulation.py @@ -11,6 +11,7 @@ CHUNK_METADATA_KEYS: frozenset[str] = frozenset( "duplex_epoch", "duplex_turn_id", "llm_output_text_utf8", + "segment_end", "tts_is_last_chunk", "turn_end", } diff --git a/vllm_omni/outputs/output_processor.py b/vllm_omni/outputs/output_processor.py index ebfe0689a..4d06f4124 100644 --- a/vllm_omni/outputs/output_processor.py +++ b/vllm_omni/outputs/output_processor.py @@ -523,24 +523,27 @@ class MultimodalOutputProcessor(VLLMOutputProcessor): upstream_outputs.append(eco) # Handle multimodal-only outputs (generation stages) locally. - self._process_mm_only_outputs(mm_only_outputs) + mm_request_outputs = self._process_mm_only_outputs(mm_only_outputs) # Delegate text/pooling outputs to upstream. - return super().process_outputs( + processed = super().process_outputs( upstream_outputs, engine_core_timestamp=engine_core_timestamp, iteration_stats=iteration_stats, ) + processed.request_outputs.extend(mm_request_outputs) + return processed def _process_mm_only_outputs( self, engine_core_outputs: list[EngineCoreOutput], - ) -> None: + ) -> list[OmniRequestOutput | PoolingRequestOutput]: """Handle outputs from generation stages that have no detokenizer. These cannot go through upstream process_outputs because it asserts detokenizer is not None when pooling_output is None. """ + request_outputs: list[OmniRequestOutput | PoolingRequestOutput] = [] for eco in engine_core_outputs: req_state = self.request_states.get(eco.request_id) if req_state is None or not isinstance(req_state, OmniRequestState): @@ -554,6 +557,11 @@ class MultimodalOutputProcessor(VLLMOutputProcessor): req_state.num_cached_tokens = eco.num_cached_tokens req_state.is_prefilling = False + is_non_final_audio_chunk = ( + finish_reason is not None + and req_state.output_kind == RequestOutputKind.DELTA + and is_non_final_delta_audio_chunk(req_state.mm_accumulated, req_state.mm_type) + ) if request_output := req_state.make_request_output( new_token_ids, None, # pooling_output @@ -564,9 +572,13 @@ class MultimodalOutputProcessor(VLLMOutputProcessor): ): if req_state.queue is not None: req_state.queue.put(request_output) + else: + request_outputs.append(request_output) - if finish_reason is not None: + is_segment_finished = bool(getattr(eco, "is_segment_finished", False)) + if finish_reason is not None and not is_segment_finished and not is_non_final_audio_chunk: self._finish_request(req_state) + return request_outputs def _update_stats_from_output( self, From 23bb4395bb334d21c482d0d2ba04c46738d5c214 Mon Sep 17 00:00:00 2001 From: iancarrasco-b10 Date: Tue, 28 Jul 2026 16:46:19 -0700 Subject: [PATCH 12/37] [Feature] Flush the speech WebSocket on input.done without closing it (#5517) Signed-off-by: iancarrasco-b10 --- docs/serving/speech_api.md | 32 +- .../examples/online_serving/text_to_speech.md | 9 +- .../online_serving/text_to_speech/README.md | 9 +- .../qwen3_tts/streaming_speech_client.py | 374 ++++++++++-------- .../openai_api/test_qwen3_tts_websocket.py | 138 +++++-- .../openai_api/test_serving_speech_stream.py | 218 +++++++++- vllm_omni/entrypoints/openai/api_server.py | 5 +- .../openai/serving_speech_stream.py | 205 ++++++---- 8 files changed, 688 insertions(+), 302 deletions(-) diff --git a/docs/serving/speech_api.md b/docs/serving/speech_api.md index 2bf630476..0deea7f8c 100644 --- a/docs/serving/speech_api.md +++ b/docs/serving/speech_api.md @@ -276,20 +276,42 @@ Client -> Server: | Message | Description | |---------|-------------| -| `{"type": "session.config", ...}` | Session configuration (sent once, first message) | +| `{"type": "session.config", ...}` | Session configuration (first message; may be resent between utterances to change it) | | `{"type": "input.text", "text": "..."}` | Text chunk | -| `{"type": "input.done"}` | End of input, flushes remaining buffer | +| `{"type": "input.done"}` | End of utterance: flushes the buffer and keeps the connection open | +| `{"type": "session.close"}` | End of connection | Server -> Client: | Message | Description | |---------|-------------| -| `{"type": "audio.start", "sentence_index": 0, "sentence_text": "...", "format": "pcm", "sample_rate": 24000}` | Audio generation starting for the buffered input | +| `{"type": "audio.start", "utterance_index": 0, "sentence_index": 0, "sentence_text": "...", "format": "pcm", "sample_rate": 24000}` | Audio generation starting for the buffered input | | Binary frame | Raw audio bytes (one or more PCM chunks when `stream_audio=true`) | -| `{"type": "audio.done", "sentence_index": 0, "total_bytes": 96000, "error": false}` | Audio complete for the buffered input | -| `{"type": "session.done", "total_sentences": N}` | Session complete | +| `{"type": "audio.done", "utterance_index": 0, "sentence_index": 0, "total_bytes": 96000, "error": false}` | Audio complete for the buffered input | +| `{"type": "session.done", "utterance_index": 0, "total_sentences": N}` | Flushed utterance complete | | `{"type": "error", "message": "..."}` | Non-fatal error | +### Flushing vs. Closing + +`input.done` is a flush, not a disconnect. The server synthesizes the buffered +text, emits `session.done`, and then waits on the same connection for the next +utterance, so a client that speaks repeatedly (for example one driven by an +upstream LLM) pays the WebSocket handshake once instead of once per utterance. + +* The session config is sticky. Send `input.text` again straight after + `session.done` to reuse it, or send another `session.config` first to change + voice, format, or reference audio. A `session.config` sent while text is + still buffered is rejected so no pending input is silently dropped. +* An utterance is the flush unit, not a linguistic one: it is whatever text was + buffered when `input.done` arrived, of any length, synthesized as one request. + `utterance_index` counts those flushes across the connection, so it tells you + which `input.done` a frame belongs to. `sentence_index` counts within one + flush and so pairs with `total_sentences`, which means every utterance reports + `sentence_index: 0` of `total_sentences: 1` (or `0` for an empty buffer). +* End the connection with `session.close`, or by closing the socket. An idle + connection is still closed after the server's idle timeout, which now also + applies to the gap between utterances. + ### Session Config Parameters All REST API parameters are supported, plus: diff --git a/docs/user_guide/examples/online_serving/text_to_speech.md b/docs/user_guide/examples/online_serving/text_to_speech.md index 5640845d8..c6a613723 100644 --- a/docs/user_guide/examples/online_serving/text_to_speech.md +++ b/docs/user_guide/examples/online_serving/text_to_speech.md @@ -422,12 +422,19 @@ curl -X POST http://localhost:8091/v1/audio/speech \ Raw PCM streaming requires `stream_format="audio"`, `response_format="pcm"`, and `async_chunk: true` on the stage config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming. ### Streaming WebSocket -The `/v1/audio/speech/stream` endpoint accepts text incrementally, splits it at sentence boundaries, and emits one PCM stream per sentence: +The `/v1/audio/speech/stream` endpoint accepts text incrementally and synthesizes the buffered text as one continuous request on `input.done`: ```bash python qwen3_tts/streaming_speech_client.py --text "Hello world. How are you? I am fine." python qwen3_tts/streaming_speech_client.py --text "..." --simulate-stt --stt-delay 0.1 ``` +`input.done` flushes without closing, so repeating `--text` synthesizes several utterances over one connection: +```bash +python qwen3_tts/streaming_speech_client.py \ + --text "First utterance." \ + --text "Second utterance, same connection." +``` + ### Gradio demos ```bash ./qwen3_tts/run_gradio_demo.sh # CustomVoice (default) diff --git a/examples/online_serving/text_to_speech/README.md b/examples/online_serving/text_to_speech/README.md index 201cb5d78..7db0a1844 100644 --- a/examples/online_serving/text_to_speech/README.md +++ b/examples/online_serving/text_to_speech/README.md @@ -537,12 +537,19 @@ curl -X POST http://localhost:8091/v1/audio/speech \ Raw PCM streaming requires `stream_format="audio"`, `response_format="pcm"`, and `async_chunk: true` on the stage config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming. ### Streaming WebSocket -The `/v1/audio/speech/stream` endpoint accepts text incrementally, splits it at sentence boundaries, and emits one PCM stream per sentence: +The `/v1/audio/speech/stream` endpoint accepts text incrementally and synthesizes the buffered text as one continuous request on `input.done`: ```bash python qwen3_tts/streaming_speech_client.py --text "Hello world. How are you? I am fine." python qwen3_tts/streaming_speech_client.py --text "..." --simulate-stt --stt-delay 0.1 ``` +`input.done` flushes without closing, so repeating `--text` synthesizes several utterances over one connection: +```bash +python qwen3_tts/streaming_speech_client.py \ + --text "First utterance." \ + --text "Second utterance, same connection." +``` + To receive word-level timestamps, launch the server with a forced aligner: ```bash vllm-omni serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \ diff --git a/examples/online_serving/text_to_speech/qwen3_tts/streaming_speech_client.py b/examples/online_serving/text_to_speech/qwen3_tts/streaming_speech_client.py index 92a40739e..e141a4c48 100644 --- a/examples/online_serving/text_to_speech/qwen3_tts/streaming_speech_client.py +++ b/examples/online_serving/text_to_speech/qwen3_tts/streaming_speech_client.py @@ -7,6 +7,12 @@ Usage: # Send full text at once python streaming_speech_client.py --text "Hello world. How are you? I am fine." + # Several utterances over one connection: input.done flushes each of them + # and the connection stays open, so only the first pays the handshake + python streaming_speech_client.py \ + --text "First utterance." \ + --text "Second utterance, same connection." + # Pick a built-in speaker for CustomVoice models python streaming_speech_client.py \ --text "打开电子枪，关闭扫描，切换到低倍模式。" \ @@ -64,18 +70,27 @@ except ImportError: raise SystemExit(1) -def save_audio_file(output_dir: str, sentence_index: int, response_format: str, chunks: list[bytes]) -> str: +def frame_basename(msg: dict) -> str: + """Name a file after the utterance and sentence a frame belongs to. + + Every utterance is sentence 0, so the utterance index is what keeps the + files of one connection from overwriting each other. + """ + return f"utterance_{msg['utterance_index']:03d}_sentence_{msg['sentence_index']:03d}" + + +def save_audio_file(output_dir: str, basename: str, response_format: str, chunks: list[bytes]) -> str: audio_bytes = b"".join(chunks) if response_format == "pcm": - raw_path = os.path.join(output_dir, f"sentence_{sentence_index:03d}.pcm") + raw_path = os.path.join(output_dir, f"{basename}.pcm") with open(raw_path, "wb") as f: f.write(audio_bytes) - wav_path = os.path.join(output_dir, f"sentence_{sentence_index:03d}.wav") + wav_path = os.path.join(output_dir, f"{basename}.wav") save_pcm_wav(wav_path, audio_bytes) return wav_path - filename = os.path.join(output_dir, f"sentence_{sentence_index:03d}.{response_format}") + filename = os.path.join(output_dir, f"{basename}.{response_format}") with open(filename, "wb") as f: f.write(audio_bytes) return filename @@ -95,183 +110,207 @@ def timestamp_words_text(timestamps: list[dict] | None) -> str: return " ".join(str(item.get("word", "")) for item in timestamps if item.get("word")) +async def send_utterance(ws, text: str, simulate_stt: bool, stt_delay: float) -> None: + """Send one utterance's text, then input.done to flush it.""" + if simulate_stt: + words = text.split(" ") + for i, word in enumerate(words): + chunk = word + (" " if i < len(words) - 1 else "") + await ws.send( + json.dumps( + { + "type": "input.text", + "text": chunk, + } + ) + ) + print(f" Sent: {chunk!r}") + await asyncio.sleep(stt_delay) + else: + await ws.send( + json.dumps( + { + "type": "input.text", + "text": text, + } + ) + ) + print(f"Sent full text: {text!r}") + + await ws.send(json.dumps({"type": "input.done"})) + print("Sent input.done") + + +async def receive_utterance( + ws, + output_dir: str, + response_format: str, + word_timestamps: bool, + save_chunks: bool, +) -> None: + """Consume frames until session.done marks the flushed utterance complete.""" + current_label = "utterance 0" + current_sentence_text = "" + current_chunks: list[bytes] = [] + current_timestamps: list[dict] = [] + # Distinguish `null` (aligner failed) from `[]` (silence / no tokens): + # the sentence is a failure only if no frame ever carried timestamps. + alignment_frame_seen = False + + while True: + message = await ws.recv() + + if isinstance(message, bytes): + current_chunks.append(message) + print(f" Received audio chunk for {current_label}: {len(message)} bytes") + else: + # JSON frame + msg = json.loads(message) + msg_type = msg.get("type") + + if msg_type == "audio.start": + current_label = f"utterance {msg['utterance_index']}" + current_sentence_text = msg.get("sentence_text", "") + current_chunks = [] + current_timestamps = [] + alignment_frame_seen = False + print(f" [{current_label}] Generating: {current_sentence_text!r}") + elif msg_type == "audio.chunk": + audio = base64.b64decode(msg["audio_b64"]) + current_chunks.append(audio) + chunk_timestamps = msg.get("timestamps") + # Sentence-level alignment sends a trailing timestamp-only + # frame with empty audio; don't write an empty chunk file. + if save_chunks and audio: + chunk_base = os.path.join( + output_dir, + f"{frame_basename(msg)}_chunk_{msg['chunk_id']:03d}", + ) + chunk_pcm = f"{chunk_base}.pcm" + chunk_wav = f"{chunk_base}.wav" + chunk_json = f"{chunk_base}_timestamps.json" + with open(chunk_pcm, "wb") as f: + f.write(audio) + save_pcm_wav(chunk_wav, audio, int(msg.get("sample_rate", 24000))) + with open(chunk_json, "w", encoding="utf-8") as f: + json.dump( + { + "utterance_index": msg["utterance_index"], + "sentence_index": msg["sentence_index"], + "chunk_id": msg["chunk_id"], + "chunk_start_ms": msg.get("chunk_start_ms"), + "chunk_end_ms": msg.get("chunk_end_ms"), + "sentence_text": current_sentence_text, + "covered_text": timestamp_words_text(chunk_timestamps), + "timestamps": chunk_timestamps, + }, + f, + ensure_ascii=False, + indent=2, + ) + if chunk_timestamps is None: + print(f" [{current_label}] chunk {msg['chunk_id']}: {len(audio)} bytes, timestamps unavailable") + else: + alignment_frame_seen = True + current_timestamps.extend(chunk_timestamps) + print( + f" [{current_label}] chunk {msg['chunk_id']}: " + f"{len(audio)} bytes, {len(chunk_timestamps)} timestamp(s)" + ) + if save_chunks and audio: + print(f" saved chunk -> {chunk_wav}") + elif msg_type == "audio.done": + filename = save_audio_file( + output_dir, + frame_basename(msg), + response_format, + current_chunks, + ) + if word_timestamps: + # `null` on failure, else the accumulated list (`[]` for silence). + sentence_timestamps = current_timestamps if alignment_frame_seen else None + ts_filename = os.path.join( + output_dir, + f"{frame_basename(msg)}_timestamps.json", + ) + with open(ts_filename, "w", encoding="utf-8") as f: + json.dump( + { + "utterance_index": msg["utterance_index"], + "sentence_index": msg["sentence_index"], + "sentence_text": current_sentence_text, + "covered_text": timestamp_words_text(sentence_timestamps), + "timestamps": sentence_timestamps, + }, + f, + ensure_ascii=False, + indent=2, + ) + print( + f" [{current_label}] Done" + f" bytes={msg.get('total_bytes', len(b''.join(current_chunks)))}" + f" error={msg.get('error', False)}" + f" -> {filename}" + ) + if word_timestamps: + print(f" [{current_label}] Timestamps -> {ts_filename}") + current_chunks = [] + current_timestamps = [] + alignment_frame_seen = False + elif msg_type == "session.done": + print(f"\nUtterance {msg['utterance_index']} complete: {msg['total_sentences']} sentence(s) generated") + return + elif msg_type == "error": + print(f" ERROR: {msg['message']}") + else: + print(f" Unknown message: {msg}") + + async def stream_tts( url: str, - text: str, + texts: list[str], config: dict, output_dir: str, simulate_stt: bool = False, stt_delay: float = 0.1, save_chunks: bool = False, ) -> None: - """Connect to the streaming TTS endpoint and process audio responses.""" + """Connect to the streaming TTS endpoint and process audio responses. + + Every text is synthesized over the same connection: input.done flushes an + utterance without closing, so only the first one pays the handshake. + """ os.makedirs(output_dir, exist_ok=True) + response_format = config.get("response_format", "wav") + word_timestamps = bool(config.get("word_timestamps", False)) async with websockets.connect(url) as ws: - # 1. Send session config + # 1. Send session config; it stays in effect for every utterance below. config_msg = {"type": "session.config", **config} await ws.send(json.dumps(config_msg)) print(f"Sent session config: {config}") - # 2. Send text (either all at once or word-by-word) - async def send_text(): - if simulate_stt: - words = text.split(" ") - for i, word in enumerate(words): - chunk = word + (" " if i < len(words) - 1 else "") - await ws.send( - json.dumps( - { - "type": "input.text", - "text": chunk, - } - ) - ) - print(f" Sent: {chunk!r}") - await asyncio.sleep(stt_delay) - else: - await ws.send( - json.dumps( - { - "type": "input.text", - "text": text, - } - ) - ) - print(f"Sent full text: {text!r}") - - # 3. Signal end of input - await ws.send(json.dumps({"type": "input.done"})) - print("Sent input.done") - - # Run sender and receiver concurrently - sender_task = asyncio.create_task(send_text()) - - response_format = config.get("response_format", "wav") - word_timestamps = bool(config.get("word_timestamps", False)) - current_sentence_index = 0 - current_sentence_text = "" - current_chunks: list[bytes] = [] - current_timestamps: list[dict] = [] - # Distinguish `null` (aligner failed) from `[]` (silence / no tokens): - # the sentence is a failure only if no frame ever carried timestamps. - alignment_frame_seen = False - - try: - while True: - message = await ws.recv() - - if isinstance(message, bytes): - current_chunks.append(message) - print(f" Received audio chunk for sentence {current_sentence_index}: {len(message)} bytes") - else: - # JSON frame - msg = json.loads(message) - msg_type = msg.get("type") - - if msg_type == "audio.start": - current_sentence_index = msg["sentence_index"] - current_sentence_text = msg.get("sentence_text", "") - current_chunks = [] - current_timestamps = [] - alignment_frame_seen = False - print(f" [sentence {msg['sentence_index']}] Generating: {current_sentence_text!r}") - elif msg_type == "audio.chunk": - audio = base64.b64decode(msg["audio_b64"]) - current_chunks.append(audio) - chunk_timestamps = msg.get("timestamps") - # Sentence-level alignment sends a trailing timestamp-only - # frame with empty audio; don't write an empty chunk file. - if save_chunks and audio: - chunk_base = os.path.join( - output_dir, - f"sentence_{msg['sentence_index']:03d}_chunk_{msg['chunk_id']:03d}", - ) - chunk_pcm = f"{chunk_base}.pcm" - chunk_wav = f"{chunk_base}.wav" - chunk_json = f"{chunk_base}_timestamps.json" - with open(chunk_pcm, "wb") as f: - f.write(audio) - save_pcm_wav(chunk_wav, audio, int(msg.get("sample_rate", 24000))) - with open(chunk_json, "w", encoding="utf-8") as f: - json.dump( - { - "sentence_index": msg["sentence_index"], - "chunk_id": msg["chunk_id"], - "chunk_start_ms": msg.get("chunk_start_ms"), - "chunk_end_ms": msg.get("chunk_end_ms"), - "sentence_text": current_sentence_text, - "covered_text": timestamp_words_text(chunk_timestamps), - "timestamps": chunk_timestamps, - }, - f, - ensure_ascii=False, - indent=2, - ) - if chunk_timestamps is None: - print( - f" [sentence {msg['sentence_index']}] chunk {msg['chunk_id']}: " - f"{len(audio)} bytes, timestamps unavailable" - ) - else: - alignment_frame_seen = True - current_timestamps.extend(chunk_timestamps) - print( - f" [sentence {msg['sentence_index']}] chunk {msg['chunk_id']}: " - f"{len(audio)} bytes, {len(chunk_timestamps)} timestamp(s)" - ) - if save_chunks and audio: - print(f" saved chunk -> {chunk_wav}") - elif msg_type == "audio.done": - filename = save_audio_file( - output_dir, - msg["sentence_index"], - response_format, - current_chunks, - ) - if word_timestamps: - # `null` on failure, else the accumulated list (`[]` for silence). - sentence_timestamps = current_timestamps if alignment_frame_seen else None - ts_filename = os.path.join( - output_dir, - f"sentence_{msg['sentence_index']:03d}_timestamps.json", - ) - with open(ts_filename, "w", encoding="utf-8") as f: - json.dump( - { - "sentence_index": msg["sentence_index"], - "sentence_text": current_sentence_text, - "covered_text": timestamp_words_text(sentence_timestamps), - "timestamps": sentence_timestamps, - }, - f, - ensure_ascii=False, - indent=2, - ) - print( - f" [sentence {msg['sentence_index']}] Done" - f" bytes={msg.get('total_bytes', len(b''.join(current_chunks)))}" - f" error={msg.get('error', False)}" - f" -> {filename}" - ) - if word_timestamps: - print(f" [sentence {msg['sentence_index']}] Timestamps -> {ts_filename}") - current_chunks = [] - current_timestamps = [] - alignment_frame_seen = False - elif msg_type == "session.done": - print(f"\nSession complete: {msg['total_sentences']} sentence(s) generated") - break - elif msg_type == "error": - print(f" ERROR: {msg['message']}") - else: - print(f" Unknown message: {msg}") - finally: - sender_task.cancel() + for text in texts: + # 2. Send text (either all at once or word-by-word), then flush. + sender_task = asyncio.create_task(send_utterance(ws, text, simulate_stt, stt_delay)) try: - await sender_task - except asyncio.CancelledError: - pass # Task cancellation is expected during shutdown + await receive_utterance( + ws, + output_dir=output_dir, + response_format=response_format, + word_timestamps=word_timestamps, + save_chunks=save_chunks, + ) + finally: + sender_task.cancel() + try: + await sender_task + except asyncio.CancelledError: + pass # Task cancellation is expected during shutdown + + # 3. Release the connection once there is nothing left to say. + await ws.send(json.dumps({"type": "session.close"})) + print("Sent session.close") print(f"\nAudio files saved to: {output_dir}/") @@ -286,7 +325,8 @@ def main(): parser.add_argument( "--text", required=True, - help="Text to synthesize", + action="append", + help="Text to synthesize; repeat to send several utterances over one connection", ) parser.add_argument( "--output-dir", @@ -385,7 +425,7 @@ def main(): asyncio.run( stream_tts( url=args.url, - text=args.text, + texts=args.text, config=config, output_dir=args.output_dir, simulate_stt=args.simulate_stt, diff --git a/tests/entrypoints/openai_api/test_qwen3_tts_websocket.py b/tests/entrypoints/openai_api/test_qwen3_tts_websocket.py index ecae21158..132d43858 100644 --- a/tests/entrypoints/openai_api/test_qwen3_tts_websocket.py +++ b/tests/entrypoints/openai_api/test_qwen3_tts_websocket.py @@ -33,26 +33,63 @@ tts_ws_server_params = [ ] -async def _run_ws_session(host: str, port: int, model: str) -> dict: - uri = f"ws://{host}:{port}/v1/audio/speech/stream" +def _session_config(model: str) -> str: + return json.dumps( + { + "type": "session.config", + "model": model, + "voice": "vivian", + "language": "English", + "response_format": "pcm", + "stream_audio": True, + } + ) + + +async def _collect_utterance(ws) -> dict: + """Read frames until session.done closes out one flushed utterance.""" starts: list[dict] = [] dones: list[dict] = [] chunk_lengths: dict[int, list[int]] = {} session_done: dict | None = None + while True: + message = await asyncio.wait_for(ws.recv(), timeout=180) + if isinstance(message, bytes): + if not starts: + raise AssertionError("Received audio bytes before audio.start") + sentence_index = starts[-1]["sentence_index"] + chunk_lengths.setdefault(sentence_index, []).append(len(message)) + continue + + payload = json.loads(message) + msg_type = payload.get("type") + if msg_type == "audio.start": + starts.append(payload) + chunk_lengths.setdefault(payload["sentence_index"], []) + elif msg_type == "audio.done": + dones.append(payload) + elif msg_type == "session.done": + session_done = payload + break + elif msg_type == "error": + raise AssertionError(f"WebSocket error: {payload['message']}") + else: + raise AssertionError(f"Unexpected WebSocket message: {payload}") + + return { + "starts": starts, + "dones": dones, + "chunk_lengths": chunk_lengths, + "session_done": session_done, + } + + +async def _run_ws_session(host: str, port: int, model: str) -> dict: + uri = f"ws://{host}:{port}/v1/audio/speech/stream" + async with websockets.connect(uri, max_size=None) as ws: - await ws.send( - json.dumps( - { - "type": "session.config", - "model": model, - "voice": "vivian", - "language": "English", - "response_format": "pcm", - "stream_audio": True, - } - ) - ) + await ws.send(_session_config(model)) await ws.send( json.dumps( { @@ -67,36 +104,23 @@ async def _run_ws_session(host: str, port: int, model: str) -> dict: ) await ws.send(json.dumps({"type": "input.done"})) - while True: - message = await asyncio.wait_for(ws.recv(), timeout=180) - if isinstance(message, bytes): - if not starts: - raise AssertionError("Received audio bytes before audio.start") - sentence_index = starts[-1]["sentence_index"] - chunk_lengths.setdefault(sentence_index, []).append(len(message)) - continue + return await _collect_utterance(ws) - payload = json.loads(message) - msg_type = payload.get("type") - if msg_type == "audio.start": - starts.append(payload) - chunk_lengths.setdefault(payload["sentence_index"], []) - elif msg_type == "audio.done": - dones.append(payload) - elif msg_type == "session.done": - session_done = payload - break - elif msg_type == "error": - raise AssertionError(f"WebSocket error: {payload['message']}") - else: - raise AssertionError(f"Unexpected WebSocket message: {payload}") - return { - "starts": starts, - "dones": dones, - "chunk_lengths": chunk_lengths, - "session_done": session_done, - } +async def _run_ws_reused_connection(host: str, port: int, model: str, texts: list[str]) -> list[dict]: + """Synthesize several utterances over a single connection.""" + uri = f"ws://{host}:{port}/v1/audio/speech/stream" + results: list[dict] = [] + + async with websockets.connect(uri, max_size=None) as ws: + await ws.send(_session_config(model)) + for text in texts: + await ws.send(json.dumps({"type": "input.text", "text": text})) + await ws.send(json.dumps({"type": "input.done"})) + results.append(await _collect_utterance(ws)) + await ws.send(json.dumps({"type": "session.close"})) + + return results class TestQwen3TTSWebSocket: @@ -113,6 +137,7 @@ class TestQwen3TTSWebSocket: session_done = result["session_done"] assert session_done is not None + assert session_done["utterance_index"] == 0 assert session_done["total_sentences"] == 1 assert len(starts) == 1 assert len(dones) == 1 @@ -131,3 +156,32 @@ class TestQwen3TTSWebSocket: assert total_bytes > 0 assert chunk_lengths[sentence_index], f"Expected binary PCM frames for sentence {sentence_index}" assert sum(chunk_lengths[sentence_index]) == total_bytes + + @pytest.mark.advanced_model + @pytest.mark.tts + @hardware_test(res={"cuda": "L4"}, num_cards=1) + @pytest.mark.parametrize("omni_server", tts_ws_server_params, indirect=True) + def test_input_done_flushes_without_closing_connection(self, omni_server) -> None: + # input.done must flush the buffer and leave the connection usable: + # the second utterance is synthesized without a second handshake. + texts = [ + "This is the first utterance sent over the websocket connection.", + "This is the second utterance, reusing the very same connection.", + ] + results = asyncio.run(_run_ws_reused_connection(omni_server.host, omni_server.port, omni_server.model, texts)) + + assert len(results) == len(texts) + for expected_index, result in enumerate(results): + assert result["session_done"] == { + "type": "session.done", + "utterance_index": expected_index, + "total_sentences": 1, + } + assert len(result["starts"]) == 1 + assert len(result["dones"]) == 1 + # utterance_index counts the flushes of the connection, while + # sentence_index stays within the flush it belongs to. + assert result["starts"][0]["utterance_index"] == expected_index + assert result["starts"][0]["sentence_index"] == 0 + assert result["dones"][0]["error"] is False + assert result["dones"][0]["total_bytes"] > 0 diff --git a/tests/entrypoints/openai_api/test_serving_speech_stream.py b/tests/entrypoints/openai_api/test_serving_speech_stream.py index 881717363..44e84b346 100644 --- a/tests/entrypoints/openai_api/test_serving_speech_stream.py +++ b/tests/entrypoints/openai_api/test_serving_speech_stream.py @@ -72,13 +72,162 @@ class TestStreamingSpeechWebSocket: assert audio.startswith(b"RIFF") done = ws.receive_json() - assert done == {"type": "audio.done", "sentence_index": 0, "total_bytes": len(audio), "error": False} + assert done == { + "type": "audio.done", + "utterance_index": 0, + "sentence_index": 0, + "total_bytes": len(audio), + "error": False, + } session_done = ws.receive_json() - assert session_done == {"type": "session.done", "total_sentences": 1} + assert session_done == {"type": "session.done", "utterance_index": 0, "total_sentences": 1} assert speech_service._generate_audio_bytes.await_count == 1 + def test_input_done_flushes_and_keeps_connection_open(self, mocker: MockerFixture): + app, speech_service = _build_test_app(mocker=mocker) + + with TestClient(app) as client: + with client.websocket_connect("/v1/audio/speech/stream") as ws: + ws.send_json({"type": "session.config", "voice": "Vivian"}) + + # The same connection serves several utterances; the config sent + # once at the top keeps applying and utterance_index rises. + for expected_index, text in enumerate(("First utterance. ", "Second utterance. ")): + ws.send_json({"type": "input.text", "text": text}) + ws.send_json({"type": "input.done"}) + + start = ws.receive_json() + assert start["type"] == "audio.start" + assert start["utterance_index"] == expected_index + assert start["sentence_text"] == text.strip() + assert ws.receive_bytes().startswith(b"RIFF") + assert ws.receive_json()["type"] == "audio.done" + assert ws.receive_json() == { + "type": "session.done", + "utterance_index": expected_index, + "total_sentences": 1, + } + + assert speech_service._generate_audio_bytes.await_count == 2 + assert [call.args[0].voice for call in speech_service._generate_audio_bytes.await_args_list] == [ + "Vivian", + "Vivian", + ] + + def test_sentence_index_stays_within_the_flushed_utterance(self, mocker: MockerFixture): + # sentence_index counts within one flush and utterance_index counts the + # flushes, so a late utterance never reports "sentence 2 of 1". + app, _ = _build_test_app(mocker=mocker) + + with TestClient(app) as client: + with client.websocket_connect("/v1/audio/speech/stream") as ws: + ws.send_json({"type": "session.config", "voice": "Vivian"}) + + for expected_index in range(3): + ws.send_json({"type": "input.text", "text": "Hello world. "}) + ws.send_json({"type": "input.done"}) + + start = ws.receive_json() + assert start["utterance_index"] == expected_index + assert start["sentence_index"] == 0 + ws.receive_bytes() + + done = ws.receive_json() + assert done["utterance_index"] == expected_index + assert done["sentence_index"] == 0 + + session_done = ws.receive_json() + assert session_done["utterance_index"] == expected_index + assert session_done["total_sentences"] == 1 + assert start["sentence_index"] < session_done["total_sentences"] + + def test_session_config_between_utterances_replaces_config(self, mocker: MockerFixture): + app, speech_service = _build_test_app(mocker=mocker) + + with TestClient(app) as client: + with client.websocket_connect("/v1/audio/speech/stream") as ws: + for expected_index, voice in enumerate(("Vivian", "Serena")): + ws.send_json({"type": "session.config", "voice": voice}) + ws.send_json({"type": "input.text", "text": "Hello world. "}) + ws.send_json({"type": "input.done"}) + + assert ws.receive_json()["type"] == "audio.start" + ws.receive_bytes() + assert ws.receive_json()["type"] == "audio.done" + assert ws.receive_json() == { + "type": "session.done", + "utterance_index": expected_index, + "total_sentences": 1, + } + + assert [call.args[0].voice for call in speech_service._generate_audio_bytes.await_args_list] == [ + "Vivian", + "Serena", + ] + + def test_session_config_rejected_while_input_is_buffered(self, mocker: MockerFixture): + app, speech_service = _build_test_app(mocker=mocker) + + with TestClient(app) as client: + with client.websocket_connect("/v1/audio/speech/stream") as ws: + ws.send_json({"type": "session.config", "voice": "Vivian"}) + ws.send_json({"type": "input.text", "text": "Hello world. "}) + ws.send_json({"type": "session.config", "voice": "Serena"}) + + error = ws.receive_json() + assert error["type"] == "error" + assert "while input is buffered" in error["message"] + + # The buffered text survives the rejected reconfiguration. + ws.send_json({"type": "input.done"}) + assert ws.receive_json()["sentence_text"] == "Hello world." + ws.receive_bytes() + assert ws.receive_json()["type"] == "audio.done" + assert ws.receive_json() == {"type": "session.done", "utterance_index": 0, "total_sentences": 1} + + assert speech_service._generate_audio_bytes.await_args_list[0].args[0].voice == "Vivian" + + def test_session_close_ends_connection(self, mocker: MockerFixture): + app, _ = _build_test_app(mocker=mocker) + + with TestClient(app) as client: + with client.websocket_connect("/v1/audio/speech/stream") as ws: + ws.send_json({"type": "session.config", "voice": "Vivian"}) + ws.send_json({"type": "input.text", "text": "Hello world. "}) + ws.send_json({"type": "input.done"}) + + assert ws.receive_json()["type"] == "audio.start" + ws.receive_bytes() + assert ws.receive_json()["type"] == "audio.done" + assert ws.receive_json() == {"type": "session.done", "utterance_index": 0, "total_sentences": 1} + + ws.send_json({"type": "session.close"}) + with pytest.raises(WebSocketDisconnect): + ws.receive_json() + + def test_idle_timeout_closes_reused_connection(self, mocker: MockerFixture): + app, _ = _build_test_app(idle_timeout=0.05, mocker=mocker) + + with TestClient(app) as client: + with client.websocket_connect("/v1/audio/speech/stream") as ws: + ws.send_json({"type": "session.config", "voice": "Vivian"}) + ws.send_json({"type": "input.text", "text": "Hello world. "}) + ws.send_json({"type": "input.done"}) + + assert ws.receive_json()["type"] == "audio.start" + ws.receive_bytes() + assert ws.receive_json()["type"] == "audio.done" + assert ws.receive_json() == {"type": "session.done", "utterance_index": 0, "total_sentences": 1} + + # Holding a connection open is not free: an idle client still + # gets timed out after the flush. + assert ws.receive_json() == { + "type": "error", + "message": "Idle timeout: no message received", + } + def test_streaming_multiple_binary_frames(self, mocker: MockerFixture): captured_requests = [] @@ -125,9 +274,15 @@ class TestStreamingSpeechWebSocket: assert ws.receive_bytes() == b"\x06" done = ws.receive_json() - assert done == {"type": "audio.done", "sentence_index": 0, "total_bytes": 6, "error": False} + assert done == { + "type": "audio.done", + "utterance_index": 0, + "sentence_index": 0, + "total_bytes": 6, + "error": False, + } - assert ws.receive_json() == {"type": "session.done", "total_sentences": 1} + assert ws.receive_json() == {"type": "session.done", "utterance_index": 0, "total_sentences": 1} assert len(captured_requests) == 1 assert captured_requests[0].stream is True @@ -210,6 +365,8 @@ class TestStreamingSpeechWebSocket: # is fully aligned. chunk = ws.receive_json() assert chunk["type"] == "audio.chunk" + assert chunk["utterance_index"] == 0 + assert chunk["sentence_index"] == 0 assert chunk["chunk_id"] == 0 assert chunk["chunk_start_ms"] == 0 assert chunk["chunk_end_ms"] == 500 @@ -240,7 +397,13 @@ class TestStreamingSpeechWebSocket: ] done = ws.receive_json() - assert done == {"type": "audio.done", "sentence_index": 0, "total_bytes": 2000, "error": False} + assert done == { + "type": "audio.done", + "utterance_index": 0, + "sentence_index": 0, + "total_bytes": 2000, + "error": False, + } assert captured_requests[0].word_timestamps is True assert mock_align.await_count == 1 @@ -309,11 +472,12 @@ class TestStreamingSpeechWebSocket: assert ws.receive_bytes() assert ws.receive_json() == { "type": "audio.done", + "utterance_index": 0, "sentence_index": 0, "total_bytes": 36, "error": False, } - assert ws.receive_json() == {"type": "session.done", "total_sentences": 1} + assert ws.receive_json() == {"type": "session.done", "utterance_index": 0, "total_sentences": 1} def test_invalid_streaming_config(self, mocker: MockerFixture): app, _ = _build_test_app(mocker=mocker) @@ -341,7 +505,7 @@ class TestStreamingSpeechWebSocket: ws.send_json({"type": "input.text", "text": ""}) ws.send_json({"type": "input.done"}) - assert ws.receive_json() == {"type": "session.done", "total_sentences": 0} + assert ws.receive_json() == {"type": "session.done", "utterance_index": 0, "total_sentences": 0} assert speech_service._generate_audio_bytes.await_count == 0 @@ -361,11 +525,12 @@ class TestStreamingSpeechWebSocket: ws.receive_bytes() assert ws.receive_json() == { "type": "audio.done", + "utterance_index": 0, "sentence_index": 0, "total_bytes": 36, "error": False, } - assert ws.receive_json() == {"type": "session.done", "total_sentences": 1} + assert ws.receive_json() == {"type": "session.done", "utterance_index": 0, "total_sentences": 1} def test_unknown_message_type_keeps_session_open(self, mocker: MockerFixture): app, _ = _build_test_app(mocker=mocker) @@ -384,12 +549,13 @@ class TestStreamingSpeechWebSocket: ws.receive_bytes() assert ws.receive_json() == { "type": "audio.done", + "utterance_index": 0, "sentence_index": 0, "total_bytes": 36, "error": False, } - assert ws.receive_json() == {"type": "session.done", "total_sentences": 1} + assert ws.receive_json() == {"type": "session.done", "utterance_index": 0, "total_sentences": 1} def test_config_timeout_closes_session(self, mocker: MockerFixture): app, _ = _build_test_app(config_timeout=0.01, mocker=mocker) @@ -416,10 +582,19 @@ class TestStreamingSpeechWebSocket: ws.send_json({"type": "input.done"}) assert ws.receive_json()["type"] == "audio.start" - assert ws.receive_json() == {"type": "error", "message": "Generation failed for sentence 0: boom"} - assert ws.receive_json() == {"type": "audio.done", "sentence_index": 0, "total_bytes": 0, "error": True} + assert ws.receive_json() == { + "type": "error", + "message": "Generation failed for utterance 0, sentence 0: boom", + } + assert ws.receive_json() == { + "type": "audio.done", + "utterance_index": 0, + "sentence_index": 0, + "total_bytes": 0, + "error": True, + } - assert ws.receive_json() == {"type": "session.done", "total_sentences": 1} + assert ws.receive_json() == {"type": "session.done", "utterance_index": 0, "total_sentences": 1} def test_streaming_generation_error_marks_audio_done(self, mocker: MockerFixture): speech_service = mocker.MagicMock(spec=OmniOpenAIServingSpeech) @@ -453,16 +628,17 @@ class TestStreamingSpeechWebSocket: assert ws.receive_bytes() == b"\x01\x02" assert ws.receive_json() == { "type": "error", - "message": "Generation failed for sentence 0: stream boom", + "message": "Generation failed for utterance 0, sentence 0: stream boom", } assert ws.receive_json() == { "type": "audio.done", + "utterance_index": 0, "sentence_index": 0, "total_bytes": 2, "error": True, } - assert ws.receive_json() == {"type": "session.done", "total_sentences": 1} + assert ws.receive_json() == {"type": "session.done", "utterance_index": 0, "total_sentences": 1} def test_invalid_input_text_type_returns_validation_error(self, mocker: MockerFixture): app, speech_service = _build_test_app(mocker=mocker) @@ -478,7 +654,7 @@ class TestStreamingSpeechWebSocket: } ws.send_json({"type": "input.done"}) - assert ws.receive_json() == {"type": "session.done", "total_sentences": 0} + assert ws.receive_json() == {"type": "session.done", "utterance_index": 0, "total_sentences": 0} assert speech_service._generate_audio_bytes.await_count == 0 @@ -497,7 +673,7 @@ class TestStreamingSpeechWebSocket: } ws.send_json({"type": "input.done"}) - assert ws.receive_json() == {"type": "session.done", "total_sentences": 0} + assert ws.receive_json() == {"type": "session.done", "utterance_index": 0, "total_sentences": 0} assert speech_service._generate_audio_bytes.await_count == 0 @@ -550,7 +726,15 @@ class TestStreamingSpeechWebSocket: config.word_timestamps = False with pytest.raises(WebSocketDisconnect): - asyncio.run(handler._generate_and_send(websocket, config, "Hello world.", 0)) + asyncio.run( + handler._generate_and_send( + websocket, + config, + "Hello world.", + utterance_index=0, + sentence_index=0, + ) + ) speech_service.engine_client.abort.assert_awaited_once_with("req-abort") assert websocket.send_json.await_count == 2 diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index 44225b69b..6d088f65e 100755 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -1572,8 +1572,9 @@ async def delete_voice(name: str, raw_request: Request): async def streaming_speech(websocket: WebSocket): """WebSocket endpoint for streaming text input TTS. - Accepts text incrementally, splits at sentence boundaries, and - returns audio per sentence. See serving_speech_stream.py for protocol. + Accepts text incrementally and returns audio for the buffered text on + input.done, which flushes without closing the connection. See + serving_speech_stream.py for protocol. """ handler = getattr(websocket.app.state, "openai_streaming_speech", None) if handler is None: diff --git a/vllm_omni/entrypoints/openai/serving_speech_stream.py b/vllm_omni/entrypoints/openai/serving_speech_stream.py index 28acf8679..102cfda45 100644 --- a/vllm_omni/entrypoints/openai/serving_speech_stream.py +++ b/vllm_omni/entrypoints/openai/serving_speech_stream.py @@ -3,26 +3,46 @@ Accepts text incrementally via WebSocket, buffers it until input.done, and generates audio once for the buffered input using the existing TTS pipeline. +input.done is a flush, not a close: it ends the current utterance and the +connection stays open, so the next utterance reuses the same connection +instead of paying another WebSocket handshake. A connection ends on +session.close, on the idle timeout, or when the client closes the socket. +The session config is sticky across flushes and can be replaced by sending +another session.config between utterances. + +"Utterance" here names the flush unit rather than any linguistic unit: it is +whatever text the client had buffered when it sent input.done, from a word to +several paragraphs, synthesized as a single request. + Protocol: Client -> Server: - {"type": "session.config", ...} # Session config (sent once first) + {"type": "session.config", ...} # Session config (first message; repeatable) {"type": "input.text", "text": "..."} # Text chunks - {"type": "input.done"} # End of input + {"type": "input.done"} # End of utterance, flush and keep connection open + {"type": "session.close"} # End of connection Server -> Client (default, word_timestamps=false): - {"type": "audio.start", "sentence_index": 0, "sentence_text": "...", "format": "wav"} + {"type": "audio.start", "utterance_index": 0, "sentence_index": 0, + "sentence_text": "...", "format": "wav"} ... - {"type": "audio.done", "sentence_index": 0} - {"type": "session.done", "total_sentences": N} + {"type": "audio.done", "utterance_index": 0, "sentence_index": 0} + {"type": "session.done", "utterance_index": 0, "total_sentences": N} {"type": "error", "message": "..."} + # session.done ends the flushed utterance, not the connection. An + # utterance is just the flush unit: whatever text was buffered when + # input.done arrived, of any length. utterance_index counts those + # flushes across the connection, while sentence_index counts within + # one of them and so pairs with total_sentences. Server -> Client (when word_timestamps=true): - {"type": "audio.start", "sentence_index": 0, "sentence_text": "...", "format": "pcm"} - {"type": "audio.chunk", "sentence_index": 0, "chunk_id": 0, "audio_b64": "", "timestamps": null} + {"type": "audio.start", "utterance_index": 0, "sentence_index": 0, + "sentence_text": "...", "format": "pcm"} + {"type": "audio.chunk", "utterance_index": 0, "sentence_index": 0, "chunk_id": 0, + "audio_b64": "", "timestamps": null} ... {"type": "audio.chunk", "audio_b64": "", "timestamps": [{"word", "start_ms", "end_ms"}, ...]} - {"type": "audio.done", "sentence_index": 0} + {"type": "audio.done", "utterance_index": 0, "sentence_index": 0} # Audio is JSON base64 PCM (not binary). A trailing empty-audio chunk carries the # full sentence-relative alignment. timestamps: list = aligned, [] = silence, null = failed. """ @@ -57,9 +77,11 @@ _MAX_INPUT_TEXT_MESSAGE_SIZE = 128 * 1024 class OmniStreamingSpeechHandler: """Handles WebSocket sessions for streaming text-input TTS. - Each WebSocket connection is an independent session. Text arrives - incrementally, is buffered until input.done, and audio is generated once - for the buffered input using the existing OmniOpenAIServingSpeech pipeline. + A connection carries one or more utterances. Text arrives incrementally, + is buffered until input.done, and audio is generated once for the + buffered input using the existing OmniOpenAIServingSpeech pipeline. The + connection outlives each utterance so a client can keep synthesizing on + it without reconnecting. Args: speech_service: The existing TTS serving instance (reused for @@ -79,54 +101,64 @@ class OmniStreamingSpeechHandler: self._config_timeout = config_timeout async def handle_session(self, websocket: WebSocket) -> None: - """Main session loop for a single WebSocket connection.""" + """Main loop for a single WebSocket connection. + + Serves any number of utterances: text is buffered until input.done, + which flushes it as one TTS request and then leaves the connection + open for the next one. Rejecting a message is only fatal before the + first valid session.config; afterwards the error is reported and the + connection survives. + """ await websocket.accept() + config: StreamingSpeechSessionConfig | None = None + text_parts: list[str] = [] + utterance_index = 0 + try: - # 1. Wait for session.config - config = await self._receive_config(websocket) - if config is None: - return # Error already sent, connection closing - - # Validate model if specified - if config.model and hasattr(self._speech_service, "_check_model"): - error = await self._speech_service._check_model( - OpenAICreateSpeechRequest(input="ping", model=config.model) - ) - if error is not None: - await self._send_error(websocket, str(error)) - return - - text_parts: list[str] = [] - - # 2. Receive text chunks until input.done while True: try: raw = await asyncio.wait_for( websocket.receive_text(), - timeout=self._idle_timeout, + timeout=self._config_timeout if config is None else self._idle_timeout, ) except asyncio.TimeoutError: - await self._send_error(websocket, "Idle timeout: no message received") + if config is None: + await self._send_error(websocket, "Timeout waiting for session.config") + else: + await self._send_error(websocket, "Idle timeout: no message received") return - if len(raw) > _MAX_INPUT_TEXT_MESSAGE_SIZE: - await self._send_error(websocket, "input.text message too large") - continue - - try: - msg = json.loads(raw) - except json.JSONDecodeError: - await self._send_error(websocket, "Invalid JSON message") - continue - - if not isinstance(msg, dict): - await self._send_error(websocket, "WebSocket messages must be JSON objects") + msg = await self._parse_message(websocket, raw) + if msg is None: + if config is None: + return # Malformed handshake, connection closing continue msg_type = msg.get("type") - if msg_type == "input.text": + if msg_type == "session.config": + if text_parts: + await self._send_error( + websocket, + "session.config cannot be applied while input is buffered; send input.done first", + ) + continue + new_config = await self._build_config(websocket, msg) + if new_config is None: + if config is None: + return # Error already sent, connection closing + continue # Keep serving with the previous config + config = new_config + + elif config is None: + await self._send_error( + websocket, + f"Expected session.config, got: {msg_type}", + ) + return + + elif msg_type == "input.text": text = msg.get("text", "") if not isinstance(text, str): await self._send_error(websocket, "input.text requires a string value") @@ -135,17 +167,31 @@ class OmniStreamingSpeechHandler: elif msg_type == "input.done": full_text = "".join(text_parts).strip() + text_parts.clear() total_sentences = 0 if full_text: - await self._generate_and_send(websocket, config, full_text, 0) + # However long the buffered text is, the pipeline takes + # it as one request, so every flush is sentence 0 of 1. + await self._generate_and_send( + websocket, + config, + full_text, + utterance_index=utterance_index, + sentence_index=0, + ) total_sentences = 1 await websocket.send_json( { "type": "session.done", + "utterance_index": utterance_index, "total_sentences": total_sentences, } ) + utterance_index += 1 + + elif msg_type == "session.close": + await websocket.close() return else: @@ -163,44 +209,50 @@ class OmniStreamingSpeechHandler: except Exception: logger.debug("Failed to send error to streaming speech client", exc_info=True) - async def _receive_config(self, websocket: WebSocket) -> StreamingSpeechSessionConfig | None: - """Wait for and validate the session.config message.""" - try: - raw = await asyncio.wait_for( - websocket.receive_text(), - timeout=self._config_timeout, - ) - except asyncio.TimeoutError: - await self._send_error(websocket, "Timeout waiting for session.config") - return None + async def _parse_message(self, websocket: WebSocket, raw: str) -> dict | None: + """Decode one client message, or report why it was rejected. - if len(raw) > _MAX_CONFIG_MESSAGE_SIZE: - await self._send_error(websocket, "session.config message too large") + Size limits are per message type: session.config carries ref_audio + payloads and gets the larger budget. + """ + if len(raw) > max(_MAX_CONFIG_MESSAGE_SIZE, _MAX_INPUT_TEXT_MESSAGE_SIZE): + await self._send_error(websocket, "WebSocket message too large") return None try: msg = json.loads(raw) except json.JSONDecodeError: - await self._send_error(websocket, "Invalid JSON in session.config") + await self._send_error(websocket, "Invalid JSON message") return None if not isinstance(msg, dict): - await self._send_error(websocket, "session.config must be a JSON object") + await self._send_error(websocket, "WebSocket messages must be JSON objects") return None - if msg.get("type") != "session.config": - await self._send_error( - websocket, - f"Expected session.config, got: {msg.get('type')}", - ) + if msg.get("type") == "session.config": + if len(raw) > _MAX_CONFIG_MESSAGE_SIZE: + await self._send_error(websocket, "session.config message too large") + return None + elif len(raw) > _MAX_INPUT_TEXT_MESSAGE_SIZE: + await self._send_error(websocket, "input.text message too large") return None + return msg + + async def _build_config(self, websocket: WebSocket, msg: dict) -> StreamingSpeechSessionConfig | None: + """Validate a session.config message and its model.""" try: config = StreamingSpeechSessionConfig(**{k: v for k, v in msg.items() if k != "type"}) except ValidationError as e: await self._send_error(websocket, f"Invalid session config: {e}") return None + if config.model and hasattr(self._speech_service, "_check_model"): + error = await self._speech_service._check_model(OpenAICreateSpeechRequest(input="ping", model=config.model)) + if error is not None: + await self._send_error(websocket, str(error)) + return None + return config async def _generate_and_send( @@ -208,9 +260,15 @@ class OmniStreamingSpeechHandler: websocket: WebSocket, config: StreamingSpeechSessionConfig, sentence_text: str, + *, + utterance_index: int, sentence_index: int, ) -> None: - """Generate audio for a single sentence and send it over WebSocket.""" + """Generate audio for a single sentence and send it over WebSocket. + + ``utterance_index`` identifies the flush this sentence belongs to and + ``sentence_index`` its position inside that flush. + """ response_format = config.response_format or "wav" # Reject unmet word-timestamps preconditions early with a clear reason. @@ -253,6 +311,7 @@ class OmniStreamingSpeechHandler: start_payload = { "type": "audio.start", + "utterance_index": utterance_index, "sentence_index": sentence_index, "sentence_text": sentence_text, "format": response_format, @@ -277,6 +336,7 @@ class OmniStreamingSpeechHandler: request_id=request_id, generator=generator, sentence_text=sentence_text, + utterance_index=utterance_index, sentence_index=sentence_index, language=config.language, ) @@ -298,13 +358,22 @@ class OmniStreamingSpeechHandler: raise except Exception as e: generation_failed = True - logger.error("Generation failed for sentence %d: %s", sentence_index, e) - await self._send_error(websocket, f"Generation failed for sentence {sentence_index}: {e}") + logger.error( + "Generation failed for utterance %d, sentence %d: %s", + utterance_index, + sentence_index, + e, + ) + await self._send_error( + websocket, + f"Generation failed for utterance {utterance_index}, sentence {sentence_index}: {e}", + ) finally: try: await websocket.send_json( { "type": "audio.done", + "utterance_index": utterance_index, "sentence_index": sentence_index, "total_bytes": total_bytes, "error": generation_failed, @@ -320,6 +389,7 @@ class OmniStreamingSpeechHandler: request_id: str, generator, sentence_text: str, + utterance_index: int, sentence_index: int, language: str | None = None, ) -> int: @@ -350,6 +420,7 @@ class OmniStreamingSpeechHandler: await websocket.send_json( { "type": "audio.chunk", + "utterance_index": utterance_index, "sentence_index": sentence_index, "chunk_id": chunk_id, "chunk_start_ms": chunk_start_ms, From 7fa40681ffc3285b578419158ffbed1880717747 Mon Sep 17 00:00:00 2001 From: Joshna-Medisetty Date: Tue, 28 Jul 2026 17:15:36 -0700 Subject: [PATCH 13/37] Xpu/vllm 0.26 fixes (#5476) Signed-off-by: Joshna Medisetty Co-authored-by: Chendi.Xue --- .buildkite/intel/pipeline-intel.yml | 2 +- vllm_omni/platforms/interface.py | 2 ++ vllm_omni/platforms/xpu/platform.py | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.buildkite/intel/pipeline-intel.yml b/.buildkite/intel/pipeline-intel.yml index e68f59e53..acb135101 100644 --- a/.buildkite/intel/pipeline-intel.yml +++ b/.buildkite/intel/pipeline-intel.yml @@ -10,7 +10,7 @@ steps: DOCKER_BUILDKIT: "1" # Buildkite will automatically replace this with the actual commit hash VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}" - VLLM_VERSION: "v0.25.0" + VLLM_VERSION: "v0.26.0" priority: 100 timeout_in_minutes: 60 soft_fail: false diff --git a/vllm_omni/platforms/interface.py b/vllm_omni/platforms/interface.py index c4e4c6e37..9969849d9 100644 --- a/vllm_omni/platforms/interface.py +++ b/vllm_omni/platforms/interface.py @@ -109,6 +109,7 @@ class OmniPlatform(Platform): cls, selected_backend: str | None, head_size: int, + allow_trtllm_default: bool = False, ) -> str: """Get the diffusion attention backend class path for this platform. @@ -119,6 +120,7 @@ class OmniPlatform(Platform): selected_backend: User-selected backend name (e.g., "FLASH_ATTN", "TORCH_SDPA", "SAGE_ATTN"). If None, uses platform default. head_size: Attention head size. + allow_trtllm_default: Whether TRTLLM may be chosen as the default. Returns: Fully qualified class path of the selected backend. diff --git a/vllm_omni/platforms/xpu/platform.py b/vllm_omni/platforms/xpu/platform.py index 8c19868ed..26d3dccde 100644 --- a/vllm_omni/platforms/xpu/platform.py +++ b/vllm_omni/platforms/xpu/platform.py @@ -44,7 +44,9 @@ class XPUOmniPlatform(OmniPlatform, XPUPlatform): cls, selected_backend: str | None, head_size: int, + allow_trtllm_default: bool = False, ) -> str: + # XPU has no TRTLLM backend; arg accepted for signature parity but unused. compute_capability = torch.xpu.get_device_capability() # Intel Max 1100 and 1550 will not support flash_attn currently flash_attn_supported = compute_capability["architecture"] not in [13136561920] From fa41fdf25247f0c3f2092bccc7f53ccb5d21e00c Mon Sep 17 00:00:00 2001 From: Zhang Jian Date: Wed, 29 Jul 2026 08:29:33 +0800 Subject: [PATCH 14/37] [recipe][NPU] add cosmos3-super recipe on 8xA3 (#5097) Signed-off-by: Zhang Jian --- recipes/cosmos3/Cosmos3-Super.md | 131 ++++++++++++++++++ .../models/cosmos3/transformer_cosmos3.py | 3 + 2 files changed, 134 insertions(+) diff --git a/recipes/cosmos3/Cosmos3-Super.md b/recipes/cosmos3/Cosmos3-Super.md index c692b100c..54d4d7e32 100644 --- a/recipes/cosmos3/Cosmos3-Super.md +++ b/recipes/cosmos3/Cosmos3-Super.md @@ -125,3 +125,134 @@ curl -sS -X POST http://localhost:8000/v1/videos/sync -H "Accept: video/mp4" \ under the top-level `action` field. Verified on the 64B Super under `--cfg-parallel-size 2`: async `policy` returns the predicted action (`[16, 10]`) and the rollout video reliably. + +## NPU + +### 8× Ascend910 (A2, A3) + +#### Environment + +- OS: Linux +- Python: 3.10+ +- Driver / runtime: Ascend NPU driver with CANN toolkit +- Recommended operator library: **mindie-sd** (Ascend high-performance fused + operators — enables `adalayernorm` and other fused kernels automatically upon + installation) +- vLLM version: Match the repository requirements for your checkout +- vLLM-Omni version or commit: Use the commit you are deploying from + +A pre-built Docker image is available on +[Docker Hub](https://hub.docker.com/r/vllm/vllm-omni) and +[Quay.io](https://quay.io/ascend/vllm-omni). Ensure the image tag matches your +vLLM-Omni checkout so that NPU-specific code is in sync with the container. + +#### Prerequisites + +Install the **mindie-sd** operator library to enable Ascend-optimized fused +operators (`adalayernorm`, etc.): + +```bash +git clone https://gitcode.com/Ascend/MindIE-SD.git && cd MindIE-SD + +# Comment out the tik_ops build step (not needed for this use case) +sed -i 's|^\(\s*\)source ${current_script_dir}/build_tik_ops.sh|\1# source ${current_script_dir}/build_tik_ops.sh|' build/build_ops.sh + +python setup.py bdist_wheel +cd dist +pip install mindiesd-*.whl +``` + +After installation, enable the Laser Attention kernel for significant +long-sequence speedups: + +```bash +export MINDIE_SD_FA_TYPE=ascend_laser_attention +``` + +#### Command + +```bash +export MINDIE_SD_FA_TYPE=ascend_laser_attention + +vllm serve nvidia/Cosmos3-Super \ + --omni \ + --host 0.0.0.0 --port 8000 \ + --tensor-parallel-size 8 \ + --model-class-name Cosmos3OmniDiffusersPipeline \ + --no-guardrails \ + --init-timeout 1800 +``` + +#### Verification + +Same requests as the GPU section above — all modes (T2I, T2V, I2V, V2V, +T2VS, I2VS) work identically on NPU. Quick reference with +`--no-guardrails`: + +```bash +curl http://localhost:8000/v1/models + +# T2I (1024x1024, 50 steps) +curl -sS -X POST http://localhost:8000/v1/images/generations \ + -H "Content-Type: application/json" \ + -d '{ + "model": "nvidia/Cosmos3-Super", + "prompt": "A photorealistic red sports car on a city street at golden hour, cinematic lighting.", + "size": "1024x1024", "n": 1, "response_format": "b64_json", + "num_inference_steps": 50, "guidance_scale": 7.0, "seed": 42 + }' | python3 -c "import sys,json,base64; open('cosmos3_super_t2i.png','wb').write(base64.b64decode(json.load(sys.stdin)['data'][0]['b64_json']))" + +# T2V (1280×720, 189 frames, 35 steps — official params) +curl -sS -X POST http://localhost:8000/v1/videos/sync -H "Accept: video/mp4" \ + -F "model=nvidia/Cosmos3-Super" -F "prompt=A robot arm is cleaning a plate in the kitchen" \ + -F "size=1280x720" -F "num_frames=189" -F "fps=24" -F "num_inference_steps=35" \ + -F "guidance_scale=6.0" -F "max_sequence_length=4096" -F "flow_shift=10.0" \ + -F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":false}' \ + -F "seed=17" -o cosmos3_super_t2v.mp4 + +# I2V — add an uploaded reference image +curl -sS -X POST http://localhost:8000/v1/videos/sync -H "Accept: video/mp4" \ + -F "model=nvidia/Cosmos3-Super" -F "prompt=The scene comes to life with smooth, natural motion." \ + -F "size=1280x720" -F "num_frames=189" -F "fps=24" -F "num_inference_steps=35" \ + -F "guidance_scale=6.0" -F "max_sequence_length=4096" -F "flow_shift=10.0" \ + -F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":false}' \ + -F "seed=1111" -F "input_reference=@/path/to/reference.jpg;type=image/jpeg" \ + -o cosmos3_super_i2v.mp4 + +# V2V — add an uploaded reference video. condition_video_keep can be "first" or "last". +curl -sS -X POST http://localhost:8000/v1/videos/sync -H "Accept: video/mp4" \ + -F "model=nvidia/Cosmos3-Super" -F "prompt=Continue the same scene with smooth natural motion." \ + -F "size=1280x720" -F "num_frames=189" -F "fps=24" -F "num_inference_steps=35" \ + -F "guidance_scale=6.0" -F "max_sequence_length=4096" -F "flow_shift=10.0" \ + -F 'extra_params={"condition_frame_indexes_vision":[0,1],"condition_video_keep":"first","guardrails":false}' \ + -F "seed=2222" -F "input_reference=@/path/to/reference.mp4;type=video/mp4" \ + -o cosmos3_super_v2v.mp4 + +# T2V + sound — add generate_sound/sound_duration (output muxes AAC 48 kHz stereo) +curl -sS -X POST http://localhost:8000/v1/videos/sync -H "Accept: video/mp4" \ + -F "model=nvidia/Cosmos3-Super" -F "prompt=A robot arm is cleaning a plate in the kitchen" \ + -F "size=1280x720" -F "num_frames=189" -F "fps=24" -F "num_inference_steps=35" \ + -F "guidance_scale=6.0" -F "max_sequence_length=4096" -F "flow_shift=10.0" \ + -F "generate_sound=true" -F "sound_duration=7.875" \ + -F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":false}' \ + -F "seed=17" -o cosmos3_super_t2vs.mp4 + +# I2V + sound — reference image with synchronized audio +curl -sS -X POST http://localhost:8000/v1/videos/sync -H "Accept: video/mp4" \ + -F "model=nvidia/Cosmos3-Super" -F "prompt=The scene comes to life with smooth, natural motion and ambient sound." \ + -F "size=1280x720" -F "num_frames=189" -F "fps=24" -F "num_inference_steps=35" \ + -F "guidance_scale=6.0" -F "max_sequence_length=4096" -F "flow_shift=10.0" \ + -F "generate_sound=true" -F "sound_duration=7.875" \ + -F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":false}' \ + -F "seed=1111" -F "input_reference=@/path/to/reference.jpg;type=image/jpeg" \ + -o cosmos3_super_i2vs.mp4 +``` + +#### Notes + +- **Parallelism:** `--tensor-parallel-size 8` matches the model's 8 KV heads (`num_key_value_heads: 8`, `num_attention_heads: 64` → GQA). This uses 8 davinci devices (NPU 0–3, both cores each). The remaining NPU 4–7 stay idle. +- **Model config:** 64 layers × 5120 hidden size × 64 attention heads, ~120 GB transformer on disk (27 shards). Each NPU device loads ~15 GB of model weights. +- **Peak HBM:** ~22.7 GB per device at startup (bf16 weights). Additional memory is allocated per-generation for KV cache and activations — resolution and frame count drive the peak. +- **Performance (verified on 8× Ascend910):** T2I 256² / 2 steps / guidance 1.0 → ~1.5 s. +- **Guardrails** are disabled with `--no-guardrails` (guards are on by default). The gated `nvidia/Cosmos-1.0-Guardrail` model and `cosmos-guardrail` package are not shipped. Add `guardrails: false` in `extra_params` for per-request overrides when the server has guardrails enabled. +- **Known limitations:** FP8 quantization not yet validated on Ascend NPU. `--enable-layerwise-offload` is available but untested on NPU. diff --git a/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3.py b/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3.py index 5c7e5bd8f..475c02889 100644 --- a/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3.py +++ b/vllm_omni/diffusion/models/cosmos3/transformer_cosmos3.py @@ -55,6 +55,9 @@ class RMSNorm(_VllmRMSNorm): def forward_hip(self, x: torch.Tensor) -> torch.Tensor: return self.forward_native(x) + def forward_npu(self, x: torch.Tensor) -> torch.Tensor: + return self.forward_native(x) + def _get_ulysses_state() -> tuple[int, int, dist.ProcessGroup | None]: """Return (ulysses_size, ulysses_rank, ulysses_pg) from vllm-omni parallel state. From ab0da662c069618e0cd35b309d8f41310c997d03 Mon Sep 17 00:00:00 2001 From: R0CKSTAR Date: Wed, 29 Jul 2026 10:46:27 +0800 Subject: [PATCH 15/37] [Hardware] Enable CosyVoice3 on Moore Threads MUSA (#5164) Signed-off-by: Xiaodong Ye --- .../text_to_speech/cosyvoice3/end2end.py | 7 +++++-- tests/engine/test_output_processor.py | 4 ++++ vllm_omni/deploy/cosyvoice3.yaml | 2 -- .../models/cosyvoice3_audio/cosyvoice3_dit.py | 4 ++-- .../models/cosyvoice3/cosyvoice3.py | 16 +++++++++++++++- vllm_omni/outputs/output_modality.py | 17 ++++++++++++++++- 6 files changed, 42 insertions(+), 8 deletions(-) diff --git a/examples/offline_inference/text_to_speech/cosyvoice3/end2end.py b/examples/offline_inference/text_to_speech/cosyvoice3/end2end.py index e5c266f7a..ccfb1aaef 100644 --- a/examples/offline_inference/text_to_speech/cosyvoice3/end2end.py +++ b/examples/offline_inference/text_to_speech/cosyvoice3/end2end.py @@ -182,10 +182,13 @@ def run_e2e(): print(f"Multimodal output keys: {mm.keys()}") if "audio" in mm: audio_out = mm["audio"] + # The model emits its own rate (24 kHz for CosyVoice3); take + # it from the payload so the file is not mislabeled. + sample_rate = int(mm["sr"]) if "sr" in mm.keys() else 24000 print(f"Generated Audio Shape: {audio_out.shape}") out_path = f"output_{i}.wav" - sf.write(out_path, audio_out.cpu().numpy().squeeze(), 22050) - print(f"Saved audio to {out_path}") + sf.write(out_path, audio_out.cpu().numpy().squeeze(), sample_rate) + print(f"Saved audio to {out_path} (sr={sample_rate})") else: print("No multimodal output found.") except Exception as e: diff --git a/tests/engine/test_output_processor.py b/tests/engine/test_output_processor.py index df0911aa0..81f0c8303 100644 --- a/tests/engine/test_output_processor.py +++ b/tests/engine/test_output_processor.py @@ -59,6 +59,10 @@ def _make_state(output_kind: RequestOutputKind): return OmniRequestState(**_DEFAULT_STATE_KWARGS, output_kind=output_kind) +def test_output_modality_name_uses_string_value(): + assert str(OutputModalityNames.AUDIO) == "audio" + + def test_init_empty_dict(): """Ensure mm_accumulated is initially empty.""" assert _make_state(RequestOutputKind.CUMULATIVE).mm_accumulated == {} diff --git a/vllm_omni/deploy/cosyvoice3.yaml b/vllm_omni/deploy/cosyvoice3.yaml index d81cf4a9b..551b42af6 100644 --- a/vllm_omni/deploy/cosyvoice3.yaml +++ b/vllm_omni/deploy/cosyvoice3.yaml @@ -24,8 +24,6 @@ # first. Raising connector_get_sleep_s also relaxes the consumer's polling # cadence, but trades against TTFP — keep it as a last resort. async_chunk: true -dtype: float32 - connectors: connector_of_shared_memory: name: SharedMemoryConnector diff --git a/vllm_omni/diffusion/models/cosyvoice3_audio/cosyvoice3_dit.py b/vllm_omni/diffusion/models/cosyvoice3_audio/cosyvoice3_dit.py index 6333819b0..5ae75b256 100644 --- a/vllm_omni/diffusion/models/cosyvoice3_audio/cosyvoice3_dit.py +++ b/vllm_omni/diffusion/models/cosyvoice3_audio/cosyvoice3_dit.py @@ -135,8 +135,8 @@ class DiTAttention(nn.Module): # The diffusion Attention layer expects (batch, seq, heads, head_dim) out = self.attn(query, key, value, attn_metadata=None) - # Reshape back: (batch, seq, dim) - out = out.view(batch_size, seq_len, self.inner_dim) + # Some attention backends return a non-contiguous layout. + out = out.reshape(batch_size, seq_len, self.inner_dim) out = out.to(query.dtype) # Output projection diff --git a/vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py b/vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py index b3e178565..f2f9a6a0f 100644 --- a/vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py +++ b/vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py @@ -73,6 +73,19 @@ def _cosyvoice3_trt_enabled() -> bool: return os.environ.get("COSYVOICE3_TRT", "1") not in ("0", "false", "False", "") +def _campplus_onnx_providers() -> list[str]: + """ONNX-Runtime providers for the campplus speaker-embedding session. + + Prefer ``MUSAExecutionProvider`` (from the onnxruntime-musa build) with a + CPU fallback when it is available; otherwise CPU only. The MUSA kernels + differ from the CPU reference by a few percent on the embedding, which + conditions voice cloning -- verify voice similarity if that matters. + """ + if "MUSAExecutionProvider" in onnxruntime.get_available_providers(): + return ["MUSAExecutionProvider", "CPUExecutionProvider"] + return ["CPUExecutionProvider"] + + class CosyVoice3MultiModalProcessingInfo(BaseProcessingInfo): def get_hf_config(self): """If the config is not already present pass it @@ -163,7 +176,7 @@ class CosyVoice3MultiModalProcessor(BaseMultiModalProcessor[CosyVoice3MultiModal campplus_session = onnxruntime.InferenceSession( campplus_onnx_path, sess_options=option, - providers=["CPUExecutionProvider"], + providers=_campplus_onnx_providers(), ) return { @@ -922,6 +935,7 @@ class CosyVoice3Model( if getattr(self, "_code2wav_trt_done", False): return self._code2wav_trt_done = True + if not (_cosyvoice3_trt_enabled() and torch.cuda.is_available()): return onnx_path = self._resolve_flow_estimator_onnx() diff --git a/vllm_omni/outputs/output_modality.py b/vllm_omni/outputs/output_modality.py index 2536a8ade..743669454 100644 --- a/vllm_omni/outputs/output_modality.py +++ b/vllm_omni/outputs/output_modality.py @@ -8,7 +8,22 @@ for type-safe multimodal output routing and tensor merging. from __future__ import annotations import re -from enum import Enum, Flag, StrEnum, auto +import sys +from enum import Enum, Flag, auto + +if sys.version_info >= (3, 11): + from enum import StrEnum +else: + # TODO: Remove this Python 3.10 compatibility shim when Python 3.10 support is dropped. + class StrEnum(str, Enum): + """``enum.StrEnum`` for Python 3.10, which predates it. + + ``__str__`` is taken from ``str`` so ``str(member)`` yields the member + value rather than ``"ClassName.MEMBER"``, matching 3.11 semantics. + """ + + __str__ = str.__str__ + _MODALITY_ALIASES: dict[str, str] = { "speech": "audio", From 39663d52872019fb0eb74bed8f1e811190486b0f Mon Sep 17 00:00:00 2001 From: Weiming Liao Date: Wed, 29 Jul 2026 11:21:04 +0800 Subject: [PATCH 16/37] [NPU] upgrade to v0.26.0 (#5490) Signed-off-by: Weiming Liao --- docker/Dockerfile.npu | 2 +- docker/Dockerfile.npu.a3 | 2 +- docker/Dockerfile.npu.ci | 2 +- docker/Dockerfile.npu.ci.a3 | 2 +- .../installation/npu/npu.inc.md | 12 +- vllm_omni/deploy/minicpmo_4_5_batching.yaml | 1 + vllm_omni/platforms/npu/platform.py | 2 + .../npu/worker/npu_ar_model_runner.py | 254 +++++++++------- .../npu/worker/npu_generation_model_runner.py | 276 ++++++++++-------- .../platforms/npu/worker/npu_model_runner.py | 227 +++++++++----- 10 files changed, 482 insertions(+), 298 deletions(-) diff --git a/docker/Dockerfile.npu b/docker/Dockerfile.npu index 642b89882..2dc94c2b4 100644 --- a/docker/Dockerfile.npu +++ b/docker/Dockerfile.npu @@ -1,5 +1,5 @@ ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend -ARG VLLM_ASCEND_TAG=v0.25.0 +ARG VLLM_ASCEND_TAG=v0.26.0 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG} # WORKDIR /vllm-workspace/vllm diff --git a/docker/Dockerfile.npu.a3 b/docker/Dockerfile.npu.a3 index 4fe1d56fe..246467fbe 100644 --- a/docker/Dockerfile.npu.a3 +++ b/docker/Dockerfile.npu.a3 @@ -1,5 +1,5 @@ ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend -ARG VLLM_ASCEND_TAG=v0.25.0-a3 +ARG VLLM_ASCEND_TAG=v0.26.0-a3 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG} # WORKDIR /vllm-workspace/vllm diff --git a/docker/Dockerfile.npu.ci b/docker/Dockerfile.npu.ci index 10a62fcac..c56589e9e 100644 --- a/docker/Dockerfile.npu.ci +++ b/docker/Dockerfile.npu.ci @@ -1,5 +1,5 @@ ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend -ARG VLLM_ASCEND_TAG=v0.25.0 +ARG VLLM_ASCEND_TAG=v0.26.0 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG} ARG APP_DIR=/vllm-workspace/vllm-omni diff --git a/docker/Dockerfile.npu.ci.a3 b/docker/Dockerfile.npu.ci.a3 index 5ce526c96..6a8dd4146 100644 --- a/docker/Dockerfile.npu.ci.a3 +++ b/docker/Dockerfile.npu.ci.a3 @@ -1,5 +1,5 @@ ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend -ARG VLLM_ASCEND_TAG=v0.25.0-a3 +ARG VLLM_ASCEND_TAG=v0.26.0-a3 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG} ARG APP_DIR=/vllm-workspace/vllm-omni diff --git a/docs/getting_started/installation/npu/npu.inc.md b/docs/getting_started/installation/npu/npu.inc.md index 7381f5b10..19f67cd1d 100644 --- a/docs/getting_started/installation/npu/npu.inc.md +++ b/docs/getting_started/installation/npu/npu.inc.md @@ -9,8 +9,8 @@ The recommended way to use vLLM-Omni on NPU is through the vLLM-Ascend pre-built ```bash # vLLM-Ascend image -# Atlas A2: quay.io/atlas-ci/vllm-ascend:v0.25.0 -# Atlas A3: quay.io/atlas-ci/vllm-ascend:v0.25.0-a3 +# Atlas A2: quay.io/atlas-ci/vllm-ascend:v0.26.0 +# Atlas A3: quay.io/atlas-ci/vllm-ascend:v0.26.0-a3 docker run --rm \ --name vllm-omni-npu \ --shm-size=1g \ @@ -32,7 +32,7 @@ docker run --rm \ # Inside the container, install vLLM-Omni from source cd /vllm-workspace -git clone -b v0.25.0 https://github.com/vllm-project/vllm-omni.git +git clone -b v0.26.0 https://github.com/vllm-project/vllm-omni.git cd vllm-omni pip install -v -e . --no-build-isolation # or VLLM_OMNI_TARGET_DEVICE=npu pip install -v -e . @@ -84,8 +84,8 @@ Supported images as following. Here's an example deployment command that has been verified on 4 x NPUs: ```bash -# Atlas A2: quay.io/ascend/vllm-omni:v0.25.0 -# Atlas A3: quay.io/ascend/vllm-omni:v0.25.0-a3 +# Atlas A2: quay.io/ascend/vllm-omni:v0.26.0 +# Atlas A3: quay.io/ascend/vllm-omni:v0.26.0-a3 docker run --rm \ --name vllm-omni-a3 \ --shm-size=4g \ @@ -110,7 +110,7 @@ docker run --rm \ -v /etc/ascend_install.info:/etc/ascend_install.info \ -v ~/.cache:/root/.cache \ -p 8091:8091 \ - -it quay.io/ascend/vllm-omni:v0.25.0-a3 bash + -it quay.io/ascend/vllm-omni:v0.26.0-a3 bash ``` !!! tip diff --git a/vllm_omni/deploy/minicpmo_4_5_batching.yaml b/vllm_omni/deploy/minicpmo_4_5_batching.yaml index 75693feb7..7e6b1fb63 100644 --- a/vllm_omni/deploy/minicpmo_4_5_batching.yaml +++ b/vllm_omni/deploy/minicpmo_4_5_batching.yaml @@ -84,5 +84,6 @@ platforms: cudagraph_mode: PIECEWISE - stage_id: 1 max_num_batched_tokens: 8192 + gpu_memory_utilization: 0.4 compilation_config: cudagraph_mode: PIECEWISE diff --git a/vllm_omni/platforms/npu/platform.py b/vllm_omni/platforms/npu/platform.py index 42ff0b418..240f9d576 100644 --- a/vllm_omni/platforms/npu/platform.py +++ b/vllm_omni/platforms/npu/platform.py @@ -123,7 +123,9 @@ class NPUOmniPlatform(OmniPlatform, NPUPlatform): cls, selected_backend: str | None, head_size: int, + allow_trtllm_default: bool = True, ) -> str: + # NPU has no TRTLLM backend; accepted for signature parity, ignored. from importlib.util import find_spec if selected_backend is not None: diff --git a/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py b/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py index 904974829..0ff7144a9 100644 --- a/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py +++ b/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py @@ -3,6 +3,7 @@ from __future__ import annotations +import logging import time from collections.abc import Mapping from copy import copy, deepcopy @@ -28,8 +29,8 @@ from vllm.v1.outputs import ( from vllm.v1.spec_decode.metadata import SpecDecodeMetadata from vllm.v1.structured_output.utils import apply_grammar_bitmask from vllm.v1.utils import record_function_or_nullcontext +from vllm.v1.worker import mamba_utils from vllm.v1.worker.gpu_model_runner import AsyncGPUModelRunnerOutput, PerLayerAttnMetadata -from vllm.v1.worker.mamba_utils import preprocess_mamba from vllm.v1.worker.ubatch_utils import maybe_create_ubatch_slices from vllm_ascend.ascend_forward_context import set_ascend_forward_context from vllm_ascend.attention.utils import AscendCommonAttentionMetadata @@ -43,7 +44,10 @@ from vllm_ascend.worker.model_runner_v1 import graph_capture from vllm_omni.data_entry_keys import flatten_payload from vllm_omni.distributed.omni_connectors.kv_transfer_manager import OmniKVTransferManager -from vllm_omni.distributed.omni_connectors.utils.config import stage_sends_async_output +from vllm_omni.distributed.omni_connectors.utils.config import ( + get_stage_connector_role, + stage_sends_async_output, +) from vllm_omni.outputs import OmniModelRunnerOutput from vllm_omni.platforms.npu.worker.npu_model_runner import OmniNPUModelRunner from vllm_omni.utils.mm_outputs import build_mm_cpu, partition_payload_list, to_payload_element @@ -114,6 +118,7 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): # Initialize KV cache manager (preserve vllm_config fallback behavior) self.kv_transfer_manager = OmniKVTransferManager.from_vllm_config(self.vllm_config, self.model_config) self._async_chunk = getattr(self.model_config, "async_chunk", False) + # -------------------------------------- Omni-new ------------------------------------------------- _OMNI_CONNECTOR_INIT_ARCHS = { "Qwen3OmniMoeForConditionalGeneration", "Qwen2_5OmniForConditionalGeneration", @@ -125,12 +130,17 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): "DyninOmniForConditionalGeneration", "IndexTTS2TalkerForConditionalGeneration", } - if getattr(self.model_config, "model_arch", None) in _OMNI_CONNECTOR_INIT_ARCHS: + stage_archs = set(getattr(self.model_config, "architectures", None) or ()) + model_arch_override = getattr(self.model_config, "model_arch", None) + if model_arch_override: + stage_archs.add(model_arch_override) + if stage_archs & _OMNI_CONNECTOR_INIT_ARCHS or get_stage_connector_role(self.model_config) is not None: self.init_omni_connectors( model_config=self.model_config, kv_transfer_manager=self.kv_transfer_manager, ) self._downstream_payload_cache: dict[str, bool] = {} + # -------------------------------------- Omni-new ------------------------------------------------- def _make_buffer(self, *size, dtype, numpy=True): # Prevent ray from pinning the buffer due to large size @@ -209,6 +219,35 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): model = getattr(self, "model", None) return bool(getattr(model, "requires_full_prefix_cached_hidden_states", True)) + def _deferred_prefix_cache_mm_keys(self) -> set[str]: + """Model-declared multimodal keys whose prefix-cache writes are deferred.""" + model = getattr(self, "model", None) + keys = getattr(model, "deferred_prefix_cache_mm_keys", ()) + return set(keys or ()) + + def _stage_deferred_prefix_cache_mm_outputs( + self, + *, + scheduler_output: SchedulerOutput, + multimodal_outputs: Any, + query_start_loc_cpu: Any, + ) -> None: + """See gpu_ar_model_runner._stage_deferred_prefix_cache_mm_outputs.""" + if self.omni_prefix_cache is None: + return + + deferred_mm_cache_keys = self._deferred_prefix_cache_mm_keys() + if not deferred_mm_cache_keys: + return + + self.omni_prefix_cache.stage_deferred_mm_outputs( + query_start_loc=query_start_loc_cpu, + input_batch=self.input_batch, + multimodal_outputs=flatten_payload(multimodal_outputs) if multimodal_outputs else multimodal_outputs, + num_scheduled_tokens=scheduler_output.num_scheduled_tokens, + deferred_mm_cache_keys=deferred_mm_cache_keys, + ) + def _maybe_update_prefix_cache( self, hidden_states: torch.Tensor, @@ -217,19 +256,16 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): num_tokens_padded: int, ): if self.omni_prefix_cache is not None and get_pp_group().is_last_rank: - if multimodal_outputs is not None and not isinstance(multimodal_outputs, Mapping): - logger.warning_once( - "prefix caching expects mm outputs to be a dict, but got %s", - type(multimodal_outputs), - ) - hs_for_cache = hidden_states if self._model_needs_full_prefix_hidden_states() else None + slot_mapping_gpu = self.input_batch.block_table[0].slot_mapping.gpu + slot_mapping_cpu = slot_mapping_gpu[:num_tokens_padded].cpu() self.omni_prefix_cache.update_omni_tensor_prefix_cache( hidden_states=hs_for_cache, multimodal_outputs=flatten_payload(multimodal_outputs) if multimodal_outputs else multimodal_outputs, num_tokens_unpadded=num_tokens_unpadded, - slot_mapping=self.input_batch.block_table[0].slot_mapping.cpu, + slot_mapping=slot_mapping_cpu, num_tokens_padded=num_tokens_padded, + skip_mm_cache_keys=self._deferred_prefix_cache_mm_keys(), ) def _maybe_get_combined_prefix_cache_tensors( @@ -240,6 +276,11 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): ) -> tuple[dict[str, torch.Tensor] | None, dict | None]: combined_hidden_states, combined_multimodal_outputs = None, None if self.omni_prefix_cache is not None: + if ( + not self._model_needs_full_prefix_hidden_states() + and not self.omni_prefix_cache.has_prefix_cached_new_req_ids() + ): + return None, None if self._model_needs_full_prefix_hidden_states(): combined_hidden_states = self.omni_prefix_cache.get_merged_hidden_states( query_start_loc=self.query_start_loc.cpu, @@ -358,9 +399,12 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): capturer = self.routed_experts_capturer if capturer is not None and hasattr(capturer, "finalize_pending_copy"): capturer.finalize_pending_copy() - if self.ascend_config.profiling_chunk_config.enabled: - self._sync_device() - self._execution_start_time = time.perf_counter() + if self.ascend_config.scheduler_config.profiling_chunk_config.need_timing: + if getattr(scheduler_output, "disable_profiling_timing", False): + self.ascend_config.scheduler_config.profiling_chunk_config.need_timing = False + else: + self._sync_device() + self._execution_start_time = time.perf_counter() if self.execute_model_state is not None: raise RuntimeError("State error: sample_tokens() must be called after execute_model() returns None.") @@ -407,6 +451,12 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): flush_ids.update({rid for rid in self._pending_full_payload_send if rid not in self.requests}) if flush_ids: self.flush_full_payload_outputs(flush_ids) + + if self.omni_prefix_cache is not None and scheduler_output.finished_req_ids: + self.omni_prefix_cache.commit_deferred_mm_outputs( + set(scheduler_output.finished_req_ids), + self.input_batch, + ) # self._draft_token_ids is None when `input_fits_in_drafter=False` # and there is no draft tokens scheduled. so it need to update the # spec_decoding info in scheduler_output with async_scheduling. @@ -414,20 +464,11 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): # scheduler_output in engine core process. # TODO(Ronald1995): deepcopy is expensive when there is a large # number of requests, optimize it later. - if (( + if ( self.use_async_scheduling and self.num_spec_tokens and self._draft_token_ids is None # type: ignore[has-type] - ) or ( - # NOTE: This branch specifically triggers a deepcopy during the prefill phase - # only for PCP (Parallel Context Processing) + Multi-Modal (MM) scenarios. - # It does not affect other use cases. This is a temporary workaround and - # will be removed once upstream vLLM provides native support for PCP + MM. - self.pcp_size > 1 - and self.supports_mm_inputs - and get_pp_group().is_first_rank - and not self.model_config.is_encoder_decoder - )): + ): scheduler_output = deepcopy(scheduler_output) # -------------------------------------- Omni-new ------------------------------------------------- @@ -449,6 +490,7 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): # -------------------------------------- Omni-new ------------------------------------------------- if has_ec_transfer() and get_ec_transfer().is_producer: + self._start_dump_data() with self.maybe_get_ec_connector_output( scheduler_output, encoder_cache=self.encoder_cache, @@ -458,6 +500,7 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): kv_ids = self.kv_extracted_req_ids self.kv_extracted_req_ids = None + self._finalize_dump_data() output = make_empty_encoder_model_runner_output(scheduler_output) if kv_ids: output = copy(output) @@ -498,6 +541,7 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): "it when the requests need prompt logprobs" ) + self._start_dump_data() num_reqs = self.input_batch.num_reqs req_ids = self.input_batch.req_ids tokens = [scheduler_output.num_scheduled_tokens[i] for i in req_ids] @@ -514,8 +558,6 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): ) num_tokens_unpadded = scheduler_output.total_num_scheduled_tokens - if self.pcp_size > 1: - num_tokens_unpadded = self.pcp_manager.total_num_sampled_tokens_pcp cascade_attn_prefix_lens = None # Disable cascade attention when using microbatching (DBO) if self.cascade_attn_enabled and not self.parallel_config.enable_dbo: @@ -542,14 +584,15 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): num_encoder_reqs=len(scheduler_output.scheduled_encoder_inputs), ) - logger.debug( - "Running batch with cudagraph_mode: %s, batch_descriptor: %s, " - "should_ubatch: %s, num_tokens_across_dp: %s", - cudagraph_mode, - batch_desc, - should_ubatch, - num_tokens_across_dp, - ) + if logger.isEnabledFor(logging.DEBUG): + logger.debug( + "Running batch with cudagraph_mode: %s, batch_descriptor: %s, " + "should_ubatch: %s, num_tokens_across_dp: %s", + cudagraph_mode, + batch_desc, + should_ubatch, + num_tokens_across_dp, + ) num_tokens_padded = batch_desc.num_tokens num_reqs_padded = batch_desc.num_reqs if batch_desc.num_reqs is not None else num_reqs @@ -561,6 +604,9 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): self.parallel_config.num_ubatches, ) + if self.dynamic_eplb: + self.update_eplb_heat_collection_status(num_tokens_padded) + pad_attn = cudagraph_mode == CUDAGraphMode.FULL # NOTE(Angazenn): According to https://github.com/vllm-project/vllm/pull/30877, @@ -574,7 +620,9 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): if deferred_state_corrections_fn: deferred_state_corrections_fn() deferred_state_corrections_fn = None - preprocess_mamba( + mamba_bufs = self._get_mamba_bufs() + preprocess_bufs = mamba_bufs.preprocess + mamba_utils.preprocess_mamba( scheduler_output, self.kv_cache_config, self.cache_config, @@ -583,7 +631,7 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): self.requests, self.compilation_config.static_forward_context, self.model.get_mamba_state_copy_func(), - self._get_mamba_copy_bufs(), + preprocess_bufs, ) # preprocess_mamba resets num_accepted_tokens_cpu to 1 # for requests whose state was copied to a new block. @@ -592,13 +640,23 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): self.num_accepted_tokens.np[:num_reqs] = self.input_batch.num_accepted_tokens_cpu[:num_reqs] self.num_accepted_tokens.copy_to_gpu(num_reqs) + if mamba_bufs.postprocess_align is not None: + mamba_utils.stage_postprocess_inputs_to_gpu( + mamba_bufs.postprocess_align, + scheduler_output, + self.input_batch.req_ids, + num_reqs, + self.requests, + self.mamba_state_idx, + ) + use_spec_decode = len(scheduler_output.scheduled_spec_decode_tokens) > 0 ubatch_slices_attn = ubatch_slices_padded if pad_attn else ubatch_slices if ( cudagraph_mode == CUDAGraphMode.FULL or (enable_sp() and not self.model_config.use_mla) - and self.pcp_size * self.dcp_size == 1 + and self.dcp_size == 1 ): # Currently, Graph Mode and SP will both pad num_tokens, # Another possible condition is num_tokens_padded != num_tokens_unpadded @@ -613,9 +671,7 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): ) (attn_metadata, spec_decode_common_attn_metadata) = self._build_attention_metadata( - num_tokens=num_tokens_unpadded - if not (self.use_cp and self.pcp_manager.pcp_use_hybrid_attn) - else total_num_scheduled_tokens, + num_tokens=num_tokens_unpadded, num_tokens_padded=num_tokens_padded, num_reqs=num_reqs, num_reqs_padded=num_reqs_padded, @@ -637,9 +693,7 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): ec_connector_output, ) = self._preprocess( scheduler_output, - num_tokens_padded - if not (self.use_cp and self.pcp_manager.pcp_use_hybrid_attn) - else total_num_scheduled_tokens, + num_tokens_padded, intermediate_tensors, ) @@ -670,21 +724,6 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): cudagraph_mode = CUDAGraphMode.NONE # Mark KV scales as calculated after the first forward pass self.calculate_kv_scales = False # type: ignore[has-type] - # prevent debugger is None - if self.debugger is not None: - dbg_cfg = getattr(self.debugger, "config", None) - dump_level = str(getattr(dbg_cfg, "level", "L1")).upper() if dbg_cfg is not None else "L1" - if dump_level in ("L0", "MIX"): - self.debugger.start(model=self.model) - else: - self.debugger.start() - if self.ascend_config.enable_async_exponential: - self.sampler.do_async_exponential( - b_s=logits_indices.shape[0], - head_dim=self.model_config.get_vocab_size(), - generators=self.input_batch.sampling_metadata.generators, - ) - # Encoder-decoder models can only compile the pure decode steps where no # encoder inputs are present. Use eager for the first pass. num_encoder_reqs = len(scheduler_output.scheduled_encoder_inputs) @@ -703,8 +742,10 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): batch_descriptor=batch_desc, num_actual_tokens=scheduler_output.total_num_scheduled_tokens, model_instance=self.model, - max_tokens_across_pcp=0 if self.pcp_size == 1 else self.pcp_manager.max_num_tokens_across_pcp, skip_compiled=has_encoder_input, + has_sinks=self._has_sinks, + input_ids=input_ids, + eplb_heat_collection_status=self.eplb_heat_collection_status if self.dynamic_eplb else False, ), self.maybe_get_kv_connector_output( scheduler_output, @@ -713,6 +754,8 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): ), ) as kv_connector_output, ): + if self.cache_config.mamba_cache_mode == "align": + mamba_utils.do_mamba_copy_block(preprocess_bufs) hidden_states = self._model_forward( num_tokens_padded, input_ids, positions, intermediate_tensors, inputs_embeds, **model_kwargs ) @@ -738,15 +781,6 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): aux_hidden_states = None if self.use_aux_hidden_state_outputs: hidden_states, aux_hidden_states = hidden_states - if self.pcp_size > 1: - # NOTE we must `slice` hidden_states because pcp_allgather_restore_idx - # ignores the padding from CUDA Graph. - hidden_states = self.pcp_manager.get_restore_hidden_states(hidden_states) - if aux_hidden_states is not None: - aux_hidden_states = [ - self.pcp_manager.get_restore_hidden_states(aux_hidden_states_pcp) - for aux_hidden_states_pcp in aux_hidden_states - ] # -------------------------------------- Omni-new ------------------------------------------------- self._maybe_update_prefix_cache( @@ -764,9 +798,9 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): assert isinstance(hidden_states, IntermediateTensors) hidden_states.kv_connector_output = kv_connector_output self.kv_connector_output = kv_connector_output - if self.debugger is not None: - self.debugger.stop() - self.debugger.step() + self._finalize_dump_data() + if self.dynamic_eplb: + self.eplb_updator.forward_end(self.eplb_heat_collection_status) return hidden_states if self.is_pooling_model: # Return the pooling output. @@ -774,9 +808,7 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): hidden_states, num_scheduled_tokens, num_scheduled_tokens_np, kv_connector_output ) output.kv_connector_output = kv_connector_output - if self.debugger is not None: - self.debugger.stop() - self.debugger.step() + self._finalize_dump_data() return output sample_hidden_states = hidden_states[logits_indices] @@ -885,6 +917,8 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): ) -> OmniModelRunnerOutput | AsyncModelRunnerOutput | IntermediateTensors: kv_connector_output = self.kv_connector_output self.kv_connector_output = None + pp = get_pp_group() + use_pp_spec_decode = self.speculative_config is not None and pp.world_size > 1 # -------------------------------------- Omni-new ------------------------------------------------- kv_extracted_req_ids = getattr(self, "kv_extracted_req_ids", None) @@ -897,11 +931,6 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): if self.execute_model_state is None: # Nothing to do (PP non-final rank case), output isn't used. - # receive sampled token ids from the last PP rank when using - # async scheduling + pipeline parallelism so downstream code - # (e.g., PCP input preparation) can access them. - if self.use_async_scheduling and get_pp_group().world_size > 1: - self._pp_receive_prev_sampled_token_ids_to_input_batch() if not kv_connector_output: return None # noqa # In case of PP with kv transfer, we need to pass through the @@ -992,6 +1021,23 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): ) self._copy_draft_token_ids_to_cpu(scheduler_output) + output_spec_token_ids = None + use_padded_batch = False + early_pp_padded_drafter = False + if self.speculative_config: + use_padded_batch = ( + self.speculative_config.use_eagle() + or self.speculative_config.uses_draft_model() + or self.speculative_config.uses_extract_hidden_states() + or self.speculative_config.use_ngram_gpu() + ) and not self.speculative_config.disable_padded_drafter_batch + early_pp_padded_drafter = use_pp_spec_decode and not self.use_async_scheduling and use_padded_batch + if early_pp_padded_drafter: + self._draft_token_ids = None + self._draft_token_req_ids = None + with record_function_or_nullcontext("draft_token"): + propose_draft_token_ids(sampler_output.sampled_token_ids) + ( logprobs_lists, valid_sampled_token_ids, @@ -1010,16 +1056,14 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): with record_function_or_nullcontext("draft_token"): if self.speculative_config: - use_padded_batch = ( - self.speculative_config - and (self.speculative_config.use_eagle() or self.speculative_config.uses_draft_model()) - and not self.speculative_config.disable_padded_drafter_batch - ) - if use_padded_batch: + if not early_pp_padded_drafter: + self._draft_token_ids = None + self._draft_token_req_ids = None + if use_padded_batch and not early_pp_padded_drafter: # EAGLE speculative decoding can use the GPU sampled tokens # as inputs, and does not need to wait for bookkeeping to finish. propose_draft_token_ids(sampler_output.sampled_token_ids) - if self.speculative_config and not use_padded_batch: + if not use_padded_batch: # ngram and other speculative decoding methods use the sampled # tokens on the CPU, so they are run after bookkeeping. propose_draft_token_ids(valid_sampled_token_ids) @@ -1030,6 +1074,19 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): if self.speculative_config is not None: self.finalize_kv_connector() + draft_token_ids = self._draft_token_ids if use_pp_spec_decode else None + if draft_token_ids is not None: + if isinstance(draft_token_ids, torch.Tensor): + num_draft_reqs = draft_token_ids.shape[0] + draft_ids_list = draft_token_ids[:num_draft_reqs].cpu().tolist() + draft_req_ids = self._draft_token_req_ids + else: + draft_ids_list = draft_token_ids + draft_req_ids = self.input_batch.req_ids + if draft_ids_list and draft_req_ids: + draft_by_req_id = dict(zip(draft_req_ids, draft_ids_list)) + output_spec_token_ids = [draft_by_req_id.get(req_id, []) for req_id in req_ids_output_copy] + routed_experts_lists = None if self.model_config.enable_return_routed_experts: capturer = self.routed_experts_capturer @@ -1073,6 +1130,12 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): if callable(query_start_loc_cpu): query_start_loc_cpu = query_start_loc_cpu() + self._stage_deferred_prefix_cache_mm_outputs( + scheduler_output=scheduler_output, + multimodal_outputs=multimodal_outputs, + query_start_loc_cpu=query_start_loc_cpu, + ) + pooler_output: list[dict[str, object]] | None = None if needs_pooler_payload: combined_hidden_states = None @@ -1222,21 +1285,22 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): ) model_runner_output.kv_extracted_req_ids = kv_extracted_req_ids model_runner_output.routed_experts = routed_experts_lists + model_runner_output.spec_token_ids = output_spec_token_ids with record_function_or_nullcontext("omni_output_builder:get_omni_connector_output"): model_runner_output.omni_connector_output = self.get_omni_connector_output() # -------------------------------------- Omni-new ------------------------------------------------- - if self.ascend_config.profiling_chunk_config.enabled and hasattr(self, "_execution_start_time"): + if self.ascend_config.scheduler_config.profiling_chunk_config.need_timing and hasattr( + self, "_execution_start_time" + ): self._sync_device() model_runner_output.execution_time_ms = (time.perf_counter() - self._execution_start_time) * 1000.0 if self.dynamic_eplb: with record_function_or_nullcontext("EPLB update"): - self.eplb_updator.forward_end() + self.eplb_updator.forward_end(self.eplb_heat_collection_status) - if self.debugger is not None: - self.debugger.stop() - self.debugger.step() + self._finalize_dump_data() if self.need_accepted_tokens: assert self.sampling_done_event is not None @@ -1247,14 +1311,6 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): global_stream().wait_event(self.sampling_done_event) self._update_states_after_model_execute(sampler_output.sampled_token_ids, scheduler_output) - # In async scheduling + PP, broadcast sampled token ids from the - # last PP rank so other PP ranks can receive them without going - # through the scheduler/engine IPC path. - if self.use_async_scheduling: - pp = get_pp_group() - if pp.world_size > 1 and pp.is_last_rank: - self._pp_broadcast_prev_sampled_token_ids(sampler_output.sampled_token_ids) - if not self.use_async_scheduling: return model_runner_output async_output = AsyncGPUModelRunnerOutput( diff --git a/vllm_omni/platforms/npu/worker/npu_generation_model_runner.py b/vllm_omni/platforms/npu/worker/npu_generation_model_runner.py index f4fc05574..24fa21524 100644 --- a/vllm_omni/platforms/npu/worker/npu_generation_model_runner.py +++ b/vllm_omni/platforms/npu/worker/npu_generation_model_runner.py @@ -4,6 +4,7 @@ from __future__ import annotations import gc +import logging import time from collections.abc import Mapping from copy import copy, deepcopy @@ -21,8 +22,8 @@ from vllm.utils.math_utils import cdiv from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT, AsyncModelRunnerOutput, make_empty_encoder_model_runner_output from vllm.v1.utils import record_function_or_nullcontext +from vllm.v1.worker import mamba_utils from vllm.v1.worker.gpu_model_runner import AsyncGPUModelRunnerOutput, PerLayerAttnMetadata -from vllm.v1.worker.mamba_utils import preprocess_mamba from vllm.v1.worker.ubatch_utils import maybe_create_ubatch_slices from vllm.v1.worker.utils import sanity_check_mm_encoder_outputs from vllm_ascend.ascend_forward_context import set_ascend_forward_context @@ -32,6 +33,7 @@ from vllm_ascend.ops.rotary_embedding import update_cos_sin from vllm_ascend.utils import enable_sp, lmhead_tp_enable from vllm_ascend.worker.model_runner_v1 import SEQ_LEN_WITH_MAX_PA_WORKSPACE +from vllm_omni.distributed.omni_connectors.utils.config import get_stage_connector_role from vllm_omni.outputs import OmniModelRunnerOutput from vllm_omni.platforms.npu.worker.npu_ar_model_runner import ExecuteModelState, _ensure_tensor_values from vllm_omni.platforms.npu.worker.npu_model_runner import OmniNPUModelRunner @@ -45,6 +47,10 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._async_chunk = getattr(self.model_config, "async_chunk", False) + # -------------------------------------- Omni-new ------------------------------------------------- + # Mirrors the init allowlist in gpu_generation_model_runner.py. The + # connector-role fallback keeps consumer stages whose arch is not on the + # allowlist (e.g. MiniCPM-o 4.5 stage 2) from starving on input. _OMNI_CONNECTOR_INIT_ARCHS = { "Qwen3OmniMoeForConditionalGeneration", "Qwen2_5OmniForConditionalGeneration", @@ -52,14 +58,20 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin "MiMoAudioModel", "Qwen3TTSTalkerForConditionalGeneration", "Qwen3TTSCode2Wav", + "AudexCode2Wav", + "AudexXCodec1", "CosyVoice3Model", "DyninOmniForConditionalGeneration", "IndexTTS2S2MelDecoder", } - if getattr(self.model_config, "model_arch", None) in _OMNI_CONNECTOR_INIT_ARCHS: + if ( + getattr(self.model_config, "model_arch", None) in _OMNI_CONNECTOR_INIT_ARCHS + or get_stage_connector_role(self.model_config) is not None + ): self.init_omni_connectors( model_config=self.model_config, ) + # -------------------------------------- Omni-new ------------------------------------------------- def _update_request_states(self, scheduler_output: SchedulerOutput): # remove requests @@ -98,9 +110,12 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin capturer = self.routed_experts_capturer if capturer is not None and hasattr(capturer, "finalize_pending_copy"): capturer.finalize_pending_copy() - if self.ascend_config.profiling_chunk_config.enabled: - self._sync_device() - self._execution_start_time = time.perf_counter() + if self.ascend_config.scheduler_config.profiling_chunk_config.need_timing: + if getattr(scheduler_output, "disable_profiling_timing", False): + self.ascend_config.scheduler_config.profiling_chunk_config.need_timing = False + else: + self._sync_device() + self._execution_start_time = time.perf_counter() if self.execute_model_state is not None: raise RuntimeError("State error: sample_tokens() must be called after execute_model() returns None.") # self._draft_token_ids is None when `input_fits_in_drafter=False` @@ -112,15 +127,6 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin # number of requests, optimize it later. if ( self.use_async_scheduling and self.num_spec_tokens and self._draft_token_ids is None # type: ignore[has-type] - ) or ( - # NOTE: This branch specifically triggers a deepcopy during the prefill phase - # only for PCP (Parallel Context Processing) + Multi-Modal (MM) scenarios. - # It does not affect other use cases. This is a temporary workaround and - # will be removed once upstream vLLM provides native support for PCP + MM. - self.pcp_size > 1 - and self.supports_mm_inputs - and get_pp_group().is_first_rank - and not self.model_config.is_encoder_decoder ): scheduler_output = deepcopy(scheduler_output) @@ -149,12 +155,19 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin # Update persistent batch states. deferred_state_corrections_fn = self._update_states(scheduler_output) + # -------------------------------------- Omni-new ------------------------------------------------- + if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"): + self.model.on_requests_finished(scheduler_output.finished_req_ids) + # -------------------------------------- Omni-new ------------------------------------------------- + if has_ec_transfer() and get_ec_transfer().is_producer: + self._start_dump_data() with self.maybe_get_ec_connector_output( scheduler_output, encoder_cache=self.encoder_cache, ) as ec_connector_output: self._execute_mm_encoder(scheduler_output) + self._finalize_dump_data() return self.attach_omni_connector_output( make_empty_encoder_model_runner_output(scheduler_output) ) @@ -185,6 +198,7 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin "it when the requests need prompt logprobs" ) + self._start_dump_data() num_reqs = self.input_batch.num_reqs req_ids = self.input_batch.req_ids tokens = [scheduler_output.num_scheduled_tokens[i] for i in req_ids] @@ -201,8 +215,6 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin ) num_tokens_unpadded = scheduler_output.total_num_scheduled_tokens - if self.pcp_size > 1: - num_tokens_unpadded = self.pcp_manager.total_num_sampled_tokens_pcp cascade_attn_prefix_lens = None # Disable cascade attention when using microbatching (DBO) if self.cascade_attn_enabled and not self.parallel_config.enable_dbo: @@ -229,14 +241,15 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin num_encoder_reqs=len(scheduler_output.scheduled_encoder_inputs), ) - logger.debug( - "Running batch with cudagraph_mode: %s, batch_descriptor: %s, " - "should_ubatch: %s, num_tokens_across_dp: %s", - cudagraph_mode, - batch_desc, - should_ubatch, - num_tokens_across_dp, - ) + if logger.isEnabledFor(logging.DEBUG): + logger.debug( + "Running batch with cudagraph_mode: %s, batch_descriptor: %s, " + "should_ubatch: %s, num_tokens_across_dp: %s", + cudagraph_mode, + batch_desc, + should_ubatch, + num_tokens_across_dp, + ) num_tokens_padded = batch_desc.num_tokens num_reqs_padded = batch_desc.num_reqs if batch_desc.num_reqs is not None else num_reqs @@ -248,6 +261,9 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin self.parallel_config.num_ubatches, ) + if self.dynamic_eplb: + self.update_eplb_heat_collection_status(num_tokens_padded) + pad_attn = cudagraph_mode == CUDAGraphMode.FULL # NOTE(Angazenn): According to https://github.com/vllm-project/vllm/pull/30877, @@ -261,7 +277,9 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin if deferred_state_corrections_fn: deferred_state_corrections_fn() deferred_state_corrections_fn = None - preprocess_mamba( + mamba_bufs = self._get_mamba_bufs() + preprocess_bufs = mamba_bufs.preprocess + mamba_utils.preprocess_mamba( scheduler_output, self.kv_cache_config, self.cache_config, @@ -270,7 +288,7 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin self.requests, self.compilation_config.static_forward_context, self.model.get_mamba_state_copy_func(), - self._get_mamba_copy_bufs(), + preprocess_bufs, ) # preprocess_mamba resets num_accepted_tokens_cpu to 1 # for requests whose state was copied to a new block. @@ -279,13 +297,23 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin self.num_accepted_tokens.np[:num_reqs] = self.input_batch.num_accepted_tokens_cpu[:num_reqs] self.num_accepted_tokens.copy_to_gpu(num_reqs) + if mamba_bufs.postprocess_align is not None: + mamba_utils.stage_postprocess_inputs_to_gpu( + mamba_bufs.postprocess_align, + scheduler_output, + self.input_batch.req_ids, + num_reqs, + self.requests, + self.mamba_state_idx, + ) + use_spec_decode = len(scheduler_output.scheduled_spec_decode_tokens) > 0 ubatch_slices_attn = ubatch_slices_padded if pad_attn else ubatch_slices if ( cudagraph_mode == CUDAGraphMode.FULL or (enable_sp() and not self.model_config.use_mla) - and self.pcp_size * self.dcp_size == 1 + and self.dcp_size == 1 ): # Currently, Graph Mode and SP will both pad num_tokens, # Another possible condition is num_tokens_padded != num_tokens_unpadded @@ -300,9 +328,7 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin ) (attn_metadata, spec_decode_common_attn_metadata) = self._build_attention_metadata( - num_tokens=num_tokens_unpadded - if not (self.use_cp and self.pcp_manager.pcp_use_hybrid_attn) - else total_num_scheduled_tokens, + num_tokens=num_tokens_unpadded, num_tokens_padded=num_tokens_padded, num_reqs=num_reqs, num_reqs_padded=num_reqs_padded, @@ -324,14 +350,16 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin ec_connector_output, ) = self._preprocess( scheduler_output, - num_tokens_padded - if not (self.use_cp and self.pcp_manager.pcp_use_hybrid_attn) - else total_num_scheduled_tokens, + num_tokens_padded, intermediate_tensors, ) + # -------------------------------------- Omni-new ------------------------------------------------- # [Omni] Pass token counts per request for code2wav output slicing model_kwargs["seq_token_counts"] = tokens + if getattr(self.model, "requires_request_ids", False): + model_kwargs["request_ids"] = list(req_ids) + # -------------------------------------- Omni-new ------------------------------------------------- # update global cos, sin update_cos_sin(positions) @@ -347,21 +375,6 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin cudagraph_mode = CUDAGraphMode.NONE # Mark KV scales as calculated after the first forward pass self.calculate_kv_scales = False # type: ignore[has-type] - # prevent debugger is None - if self.debugger is not None: - dbg_cfg = getattr(self.debugger, "config", None) - dump_level = str(getattr(dbg_cfg, "level", "L1")).upper() if dbg_cfg is not None else "L1" - if dump_level in ("L0", "MIX"): - self.debugger.start(model=self.model) - else: - self.debugger.start() - if self.ascend_config.enable_async_exponential: - self.sampler.do_async_exponential( - b_s=logits_indices.shape[0], - head_dim=self.model_config.get_vocab_size(), - generators=self.input_batch.sampling_metadata.generators, - ) - # Encoder-decoder models can only compile the pure decode steps where no # encoder inputs are present. Use eager for the first pass. num_encoder_reqs = len(scheduler_output.scheduled_encoder_inputs) @@ -380,14 +393,18 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin batch_descriptor=batch_desc, num_actual_tokens=scheduler_output.total_num_scheduled_tokens, model_instance=self.model, - max_tokens_across_pcp=0 if self.pcp_size == 1 else self.pcp_manager.max_num_tokens_across_pcp, skip_compiled=has_encoder_input, + has_sinks=self._has_sinks, + input_ids=input_ids, + eplb_heat_collection_status=self.eplb_heat_collection_status if self.dynamic_eplb else False, ), self.maybe_get_kv_connector_output( scheduler_output, **({"defer_finalize": not clear_kv_metadata}), ) as kv_connector_output, ): + if self.cache_config.mamba_cache_mode == "align": + mamba_utils.do_mamba_copy_block(preprocess_bufs) # -------------------------------------- Omni-new ------------------------------------------------- outputs = self._run_generation_model( num_tokens_padded=num_tokens_padded, @@ -437,11 +454,6 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin if self.execute_model_state is None: # Nothing to do (PP non-final rank case), output isn't used. - # receive sampled token ids from the last PP rank when using - # async scheduling + pipeline parallelism so downstream code - # (e.g., PCP input preparation) can access them. - if self.use_async_scheduling and get_pp_group().world_size > 1: - self._pp_receive_prev_sampled_token_ids_to_input_batch() if not kv_connector_output: return None # noqa # In case of PP with kv transfer, we need to pass through the @@ -552,17 +564,17 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin if self.speculative_config is not None: self.finalize_kv_connector() - if self.ascend_config.profiling_chunk_config.enabled and hasattr(self, "_execution_start_time"): + if self.ascend_config.scheduler_config.profiling_chunk_config.need_timing and hasattr( + self, "_execution_start_time" + ): self._sync_device() output.execution_time_ms = (time.perf_counter() - self._execution_start_time) * 1000.0 if self.dynamic_eplb: with record_function_or_nullcontext("EPLB update"): - self.eplb_updator.forward_end() + self.eplb_updator.forward_end(self.eplb_heat_collection_status) - if self.debugger is not None: - self.debugger.stop() - self.debugger.step() + self._finalize_dump_data() if not self.use_async_scheduling: return output async_output = AsyncGPUModelRunnerOutput( @@ -708,14 +720,16 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin force_has_lora=num_active_loras > 0, force_num_active_loras=num_active_loras, ) - if self.use_cp: - self.pcp_manager.init_batch_info( + if self.use_dcp: + self.dcp_manager.init_batch_info( num_scheduled_tokens, num_reqs, + self.input_batch.num_computed_tokens_cpu, + self.input_batch.num_prompt_tokens, ) if self.speculative_config: - self.pcp_manager.query_lens_pcp_full.cpu[:num_reqs] = torch.from_numpy(num_scheduled_tokens) - self.pcp_manager.query_lens_pcp_full.copy_to_gpu() + self.dcp_manager.query_lens_full.cpu[:num_reqs] = torch.from_numpy(num_scheduled_tokens) + self.dcp_manager.query_lens_full.copy_to_gpu() if cudagraph_runtime_mode is None: cudagraph_runtime_mode = _cudagraph_mode else: @@ -729,66 +743,82 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin # pad is needed if the pad of `num_tokens` is triggered inside CudagraphDispatcher num_tokens_across_dp[:] = num_tokens_padded num_scheduled_tokens = num_scheduled_tokens.repeat(num_reqs_padded) + + if self.dynamic_eplb: + self.update_eplb_heat_collection_status(num_tokens_padded) + # vllm-ascend does not support ubatch now ubatch_slices, ubatch_slices_padded = None, None attn_metadata: PerLayerAttnMetadata | None = None - # Build attention metadata for dummy_run - if self._should_build_dummy_attn_metadata(force_attention, is_profile, cudagraph_runtime_mode): - if create_mixed_batch: - raise NotImplementedError( - "create_mixed_batch is used for warmup deepgemm, vllm-ascend does not need it" - ) - self.attn_state = AscendAttentionState.DecodeOnly - if self.speculative_config and self.speculative_config.method == "mtp": - # `AscendAttentionState.SpecDecoding` is only designed for mla - if self.vllm_config.model_config.use_mla: - self.attn_state = AscendAttentionState.SpecDecoding + # Mirrors gpu_model_runner._dummy_run. + with self.synchronize_input_prep(): + # Build attention metadata for dummy_run + if self._should_build_dummy_attn_metadata(force_attention, is_profile, cudagraph_runtime_mode): + if create_mixed_batch: + raise NotImplementedError( + "create_mixed_batch is used for warmup deepgemm, vllm-ascend does not need it" + ) + self.attn_state = AscendAttentionState.DecodeOnly + if self.speculative_config and self.speculative_config.method == "mtp": + # `AscendAttentionState.SpecDecoding` is only designed for mla + if self.vllm_config.model_config.use_mla: + self.attn_state = AscendAttentionState.SpecDecoding + else: + self.attn_state = AscendAttentionState.ChunkedPrefill + # The reason why we use a fixed seq_len rather than max_query_len is that + # _npu_paged_attention_get_workspace only returns max workspace with specific + # seq_lens. We use this seq_len only when capturing graph, and still use max_query_len + # in inference. This will be removed once npu_fused_infer_attention_score + # outperforms _npu_paged_attention on all cases. + if profile_seq_lens is not None: + seq_lens = profile_seq_lens else: - self.attn_state = AscendAttentionState.ChunkedPrefill - # The reason why we use a fixed seq_len rather than max_query_len is that - # _npu_paged_attention_get_workspace only returns max workspace with specific - # seq_lens. We use this seq_len only when capturing graph, and still use max_query_len - # in inference. This will be removed once npu_fused_infer_attention_score - # outperforms _npu_paged_attention on all cases. - if profile_seq_lens is not None: - seq_lens = profile_seq_lens - else: - seq_lens = ( - SEQ_LEN_WITH_MAX_PA_WORKSPACE - if is_graph_capturing and using_paged_attention(num_tokens, self.vllm_config) - else max_query_len - ) # type: ignore[assignment] - self.optimistic_seq_lens_cpu[:num_reqs] = seq_lens - self.optimistic_seq_lens_cpu[num_reqs:].fill_(0) - self.seq_lens.copy_(self.optimistic_seq_lens_cpu, non_blocking=True) + seq_lens = ( + SEQ_LEN_WITH_MAX_PA_WORKSPACE + if is_graph_capturing and using_paged_attention(num_tokens, self.vllm_config) + else max_query_len + ) # type: ignore[assignment] + self.optimistic_seq_lens_cpu[:num_reqs] = seq_lens + self.optimistic_seq_lens_cpu[num_reqs:].fill_(0) + self.seq_lens.copy_(self.optimistic_seq_lens_cpu, non_blocking=True) - cum_num_tokens = self._get_cumsum_and_arange(num_scheduled_tokens, self.query_pos.np) - self.query_start_loc.np[1 : num_reqs_padded + 1] = cum_num_tokens - self.query_start_loc.copy_to_gpu() - if self._has_gdn: - self.gdn_query_start_loc.np[1 : num_reqs_padded + 1] = cum_num_tokens - self.gdn_query_start_loc.copy_to_gpu() + cum_num_tokens = self._get_cumsum_and_arange(num_scheduled_tokens, self.query_pos.np) + self.query_start_loc.np[1 : num_reqs_padded + 1] = cum_num_tokens + self.query_start_loc.copy_to_gpu() + if self._has_gdn: + self.gdn_query_start_loc.np[1 : num_reqs_padded + 1] = cum_num_tokens + self.gdn_query_start_loc.copy_to_gpu() - if not profile_cpp: - num_reqs_padded = self._pad_query_start_loc_for_fia( - self.query_start_loc, - num_tokens_padded, - num_reqs_padded, - num_reqs, - cudagraph_runtime_mode, - batch_desc.num_reqs, + if not profile_cpp: + num_reqs_padded = self._pad_query_start_loc_for_fia( + self.query_start_loc, + num_tokens_padded, + num_reqs_padded, + num_reqs, + cudagraph_runtime_mode, + batch_desc.num_reqs, + ) + + # Dummy graph runs do not go through _prepare_inputs(), but GDN/Mamba + # metadata reads block_table[:num_reqs_padded] below. Sync padded + # rows as well so device-side metadata does not see stale block ids. + self.input_batch.block_table.commit_block_table(num_reqs_padded) + + pad_attn = cudagraph_runtime_mode == CUDAGraphMode.FULL + attn_metadata, _ = self._build_attention_metadata( + num_tokens=num_tokens_unpadded, + num_tokens_padded=num_tokens_padded, + num_reqs=num_reqs, + num_reqs_padded=num_reqs_padded, + max_query_len=max_query_len, + ubatch_slices=ubatch_slices_padded if pad_attn else ubatch_slices, + for_cudagraph_capture=is_graph_capturing, + num_scheduled_tokens_np=num_scheduled_tokens, ) - - pad_attn = cudagraph_runtime_mode == CUDAGraphMode.FULL - attn_metadata, _ = self._build_attention_metadata( - num_tokens=num_tokens_unpadded, - num_tokens_padded=num_tokens_padded, - num_reqs=num_reqs_padded, - max_query_len=max_query_len, - ubatch_slices=ubatch_slices_padded if pad_attn else ubatch_slices, - for_cudagraph_capture=is_graph_capturing, - num_scheduled_tokens_np=num_scheduled_tokens, - ) + if not is_graph_capturing: + for kv_cache_gid in range(len(self.kv_cache_config.kv_cache_groups)): + blk_table = self.input_batch.block_table[kv_cache_gid] + blk_table.slot_mapping.gpu.fill_(-1) with self.maybe_dummy_run_with_lora( self.lora_config, @@ -827,6 +857,7 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin if hasattr(self.model, "get_dummy_runtime_additional_information"): runtime_addi = self.model.get_dummy_runtime_additional_information(num_reqs) model_kwargs["runtime_additional_information"] = runtime_addi + model_kwargs["seq_token_counts"] = [int(num_tokens_padded)] # -------------------------------------- Omni-new ------------------------------------------------- if self.uses_mrope: @@ -885,6 +916,9 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin aclgraph_runtime_mode=cudagraph_runtime_mode, batch_descriptor=batch_desc, model_instance=self.model, + has_sinks=self._has_sinks, + input_ids=input_ids, + eplb_heat_collection_status=self.eplb_heat_collection_status if self.dynamic_eplb else False, ): outputs = self.model( input_ids=input_ids, @@ -902,7 +936,7 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin # ------------------------------------------------------------------------------------------------- dummy_compute_logits(hidden_states) - if self.drafter: + if self.drafter and not profile_cpp: self.drafter.dummy_run( num_tokens=num_tokens_padded, with_prefill=with_prefill, @@ -915,10 +949,10 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin is_profile=is_profile, ) if is_profile and self.dynamic_eplb: - target = self.model.language_model if hasattr(self.model, "language_model") else self.model - target.clear_all_moe_loads() - if self.dynamic_eplb: - self.eplb_updator.forward_end() + self.eplb_updator.adaptor.clear_all_moe_loads() + if not is_profile and self.dynamic_eplb: + self.eplb_updator.forward_end(self.eplb_heat_collection_status) + self._finalize_dump_data(dump=False) return hidden_states, None diff --git a/vllm_omni/platforms/npu/worker/npu_model_runner.py b/vllm_omni/platforms/npu/worker/npu_model_runner.py index a9a8b529b..bc8e4f0a2 100644 --- a/vllm_omni/platforms/npu/worker/npu_model_runner.py +++ b/vllm_omni/platforms/npu/worker/npu_model_runner.py @@ -20,6 +20,7 @@ from vllm_ascend.ops.rotary_embedding import update_cos_sin from vllm_ascend.utils import enable_sp, lmhead_tp_enable from vllm_ascend.worker.model_runner_v1 import SEQ_LEN_WITH_MAX_PA_WORKSPACE +from vllm_omni.core.prefix_cache import OmniTensorPrefixCache from vllm_omni.model_executor.models.output_templates import OmniOutput from vllm_omni.platforms.npu._310p import is_310p from vllm_omni.worker.gpu_model_runner import OmniGPUModelRunner @@ -34,6 +35,37 @@ else: class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner): + def initialize_kv_cache(self, kv_cache_config) -> None: + """Create the omni tensor prefix cache. + + The omni prefix cache is used to store the hidden states of the prefix tokens + """ + NPUModelRunner.initialize_kv_cache(self, kv_cache_config) + if self.omni_prefix_cache is None and self.cache_config.enable_prefix_caching: + # Read num_blocks back off self.kv_cache_config: vllm-ascend + # deepcopies the config it was handed, so the value it stored is the + # authoritative one, not our caller's argument. + num_blocks = self.kv_cache_config.num_blocks + self.omni_prefix_cache = OmniTensorPrefixCache( + num_blocks=num_blocks, + block_size=self.cache_config.block_size, + hidden_size=self.model_config.get_hidden_size(), + hs_dtype=self.dtype, + ) + logger.info( + "Initialized omni prefix cache on NPU (num_blocks=%d, block_size=%d, hidden_size=%d). " + "Hidden-state cache is pinned host memory of roughly %.1f GiB; each per-token " + "multimodal output key allocates another tensor of the same block shape.", + num_blocks, + self.cache_config.block_size, + self.model_config.get_hidden_size(), + num_blocks + * self.cache_config.block_size + * self.model_config.get_hidden_size() + * self.dtype.itemsize + / (1024**3), + ) + def load_model(self, *args, **kwargs) -> None: if is_310p(): from vllm_omni.platforms.npu._310p.patch import apply_model_patches @@ -44,8 +76,37 @@ class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner): # in _pad_for_sequence_parallelism during execute_model. # This is a workaround for vllm-ascend not passing vllm_config to enable_sp(). enable_sp(self.vllm_config) + # ---------------------------------------Omni-new---------------------------------------------- + model = getattr(self, "model", None) + override_fn = None + if bool(getattr(model, "supports_sampled_token_ids_cpu_override", False)): + candidate = getattr(model, "consume_sampled_token_ids_cpu_override", None) + if callable(candidate): + override_fn = candidate + self._sampled_token_ids_cpu_override = override_fn + self._omni_query_start_loc_model_kwarg = bool(getattr(model, "supports_omni_query_start_loc", False)) self._maybe_enable_output_token_ids_for_model_sampler() self._init_talker_mtp() + self._prewarm_attention_capture_workspaces() + # ---------------------------------------Omni-new---------------------------------------------- + + def _update_states(self, scheduler_output) -> Any: + """Mirror NPUModelRunner._update_states and + call OmniGPUModelRunner._update_states to update Omni-specific states. + """ + req_data = scheduler_output.scheduled_cached_reqs + + if self.use_async_scheduling: + for i, req_id in enumerate(req_data.req_ids): + req_state = self.requests.get(req_id) + if req_state is None: + continue + if req_data.num_computed_tokens[i] < req_state.num_computed_tokens: + req_state.prev_num_draft_len = 0 + + self._apply_pp_sampled_tokens_from_scheduler_output(scheduler_output) + + return OmniGPUModelRunner._update_states(self, scheduler_output) @torch.inference_mode() def _dummy_run( @@ -130,14 +191,16 @@ class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner): force_has_lora=num_active_loras > 0, force_num_active_loras=num_active_loras, ) - if self.use_cp: - self.pcp_manager.init_batch_info( + if self.use_dcp: + self.dcp_manager.init_batch_info( num_scheduled_tokens, num_reqs, + self.input_batch.num_computed_tokens_cpu, + self.input_batch.num_prompt_tokens, ) if self.speculative_config: - self.pcp_manager.query_lens_pcp_full.cpu[:num_reqs] = torch.from_numpy(num_scheduled_tokens) - self.pcp_manager.query_lens_pcp_full.copy_to_gpu() + self.dcp_manager.query_lens_full.cpu[:num_reqs] = torch.from_numpy(num_scheduled_tokens) + self.dcp_manager.query_lens_full.copy_to_gpu() if cudagraph_runtime_mode is None: cudagraph_runtime_mode = _cudagraph_mode else: @@ -151,66 +214,81 @@ class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner): # pad is needed if the pad of `num_tokens` is triggered inside CudagraphDispatcher num_tokens_across_dp[:] = num_tokens_padded num_scheduled_tokens = num_scheduled_tokens.repeat(num_reqs_padded) + + if self.dynamic_eplb: + self.update_eplb_heat_collection_status(num_tokens_padded) + # vllm-ascend does not support ubatch now ubatch_slices, ubatch_slices_padded = None, None attn_metadata: PerLayerAttnMetadata | None = None - # Build attention metadata for dummy_run - if self._should_build_dummy_attn_metadata(force_attention, is_profile, cudagraph_runtime_mode): - if create_mixed_batch: - raise NotImplementedError( - "create_mixed_batch is used for warmup deepgemm, vllm-ascend does not need it" - ) - self.attn_state = AscendAttentionState.DecodeOnly - if self.speculative_config and self.speculative_config.method == "mtp": - # `AscendAttentionState.SpecDecoding` is only designed for mla - if self.vllm_config.model_config.use_mla: - self.attn_state = AscendAttentionState.SpecDecoding + with self.synchronize_input_prep(): + # Build attention metadata for dummy_run + if self._should_build_dummy_attn_metadata(force_attention, is_profile, cudagraph_runtime_mode): + if create_mixed_batch: + raise NotImplementedError( + "create_mixed_batch is used for warmup deepgemm, vllm-ascend does not need it" + ) + self.attn_state = AscendAttentionState.DecodeOnly + if self.speculative_config and self.speculative_config.method == "mtp": + # `AscendAttentionState.SpecDecoding` is only designed for mla + if self.vllm_config.model_config.use_mla: + self.attn_state = AscendAttentionState.SpecDecoding + else: + self.attn_state = AscendAttentionState.ChunkedPrefill + # The reason why we use a fixed seq_len rather than max_query_len is that + # _npu_paged_attention_get_workspace only returns max workspace with specific + # seq_lens. We use this seq_len only when capturing graph, and still use max_query_len + # in inference. This will be removed once npu_fused_infer_attention_score + # outperforms _npu_paged_attention on all cases. + if profile_seq_lens is not None: + seq_lens = profile_seq_lens else: - self.attn_state = AscendAttentionState.ChunkedPrefill - # The reason why we use a fixed seq_len rather than max_query_len is that - # _npu_paged_attention_get_workspace only returns max workspace with specific - # seq_lens. We use this seq_len only when capturing graph, and still use max_query_len - # in inference. This will be removed once npu_fused_infer_attention_score - # outperforms _npu_paged_attention on all cases. - if profile_seq_lens is not None: - seq_lens = profile_seq_lens - else: - seq_lens = ( - SEQ_LEN_WITH_MAX_PA_WORKSPACE - if is_graph_capturing and using_paged_attention(num_tokens, self.vllm_config) - else max_query_len - ) # type: ignore[assignment] - self.optimistic_seq_lens_cpu[:num_reqs] = seq_lens - self.optimistic_seq_lens_cpu[num_reqs:].fill_(0) - self.seq_lens.copy_(self.optimistic_seq_lens_cpu, non_blocking=True) + seq_lens = ( + SEQ_LEN_WITH_MAX_PA_WORKSPACE + if is_graph_capturing and using_paged_attention(num_tokens, self.vllm_config) + else max_query_len + ) # type: ignore[assignment] + self.optimistic_seq_lens_cpu[:num_reqs] = seq_lens + self.optimistic_seq_lens_cpu[num_reqs:].fill_(0) + self.seq_lens.copy_(self.optimistic_seq_lens_cpu, non_blocking=True) - cum_num_tokens = self._get_cumsum_and_arange(num_scheduled_tokens, self.query_pos.np) - self.query_start_loc.np[1 : num_reqs_padded + 1] = cum_num_tokens - self.query_start_loc.copy_to_gpu() - if self._has_gdn: - self.gdn_query_start_loc.np[1 : num_reqs_padded + 1] = cum_num_tokens - self.gdn_query_start_loc.copy_to_gpu() + cum_num_tokens = self._get_cumsum_and_arange(num_scheduled_tokens, self.query_pos.np) + self.query_start_loc.np[1 : num_reqs_padded + 1] = cum_num_tokens + self.query_start_loc.copy_to_gpu() + if self._has_gdn: + self.gdn_query_start_loc.np[1 : num_reqs_padded + 1] = cum_num_tokens + self.gdn_query_start_loc.copy_to_gpu() - if not profile_cpp: - num_reqs_padded = self._pad_query_start_loc_for_fia( - self.query_start_loc, - num_tokens_padded, - num_reqs_padded, - num_reqs, - cudagraph_runtime_mode, - batch_desc.num_reqs, + if not profile_cpp: + num_reqs_padded = self._pad_query_start_loc_for_fia( + self.query_start_loc, + num_tokens_padded, + num_reqs_padded, + num_reqs, + cudagraph_runtime_mode, + batch_desc.num_reqs, + ) + + # Dummy graph runs do not go through _prepare_inputs(), but GDN/Mamba + # metadata reads block_table[:num_reqs_padded] below. Sync padded + # rows as well so device-side metadata does not see stale block ids. + self.input_batch.block_table.commit_block_table(num_reqs_padded) + + pad_attn = cudagraph_runtime_mode == CUDAGraphMode.FULL + attn_metadata, _ = self._build_attention_metadata( + num_tokens=num_tokens_unpadded, + num_tokens_padded=num_tokens_padded, + num_reqs=num_reqs, + num_reqs_padded=num_reqs_padded, + max_query_len=max_query_len, + ubatch_slices=ubatch_slices_padded if pad_attn else ubatch_slices, + for_cudagraph_capture=is_graph_capturing, + num_scheduled_tokens_np=num_scheduled_tokens, ) - - pad_attn = cudagraph_runtime_mode == CUDAGraphMode.FULL - attn_metadata, _ = self._build_attention_metadata( - num_tokens=num_tokens_unpadded, - num_tokens_padded=num_tokens_padded, - num_reqs=num_reqs_padded, - max_query_len=max_query_len, - ubatch_slices=ubatch_slices_padded if pad_attn else ubatch_slices, - for_cudagraph_capture=is_graph_capturing, - num_scheduled_tokens_np=num_scheduled_tokens, - ) + if not is_graph_capturing: + for kv_cache_gid in range(len(self.kv_cache_config.kv_cache_groups)): + blk_table = self.input_batch.block_table[kv_cache_gid] + blk_table.slot_mapping.gpu.fill_(-1) with self.maybe_dummy_run_with_lora( self.lora_config, @@ -224,15 +302,24 @@ class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner): ): # Make sure padding doesn't exceed max_num_tokens assert num_tokens_padded <= self.max_num_tokens - if getattr(getattr(self, "model", None), "has_preprocess", False): - input_ids = self.input_ids.gpu[:num_tokens_padded] - inputs_embeds = self.inputs_embeds.gpu[:num_tokens_padded] - elif self.supports_mm_inputs and not self.model_config.is_encoder_decoder or self.enable_prompt_embeds: + # ---------------------------------------Omni-new---------------------------------------------- + model_kwargs = self._init_model_kwargs() + if self.supports_mm_inputs and not self.model_config.is_encoder_decoder: + input_ids, inputs_embeds = self._prepare_mm_inputs(num_tokens_padded) + model_kwargs = { + **model_kwargs, + **self._dummy_mm_kwargs(num_reqs), + } + elif self.enable_prompt_embeds: input_ids = None inputs_embeds = self.inputs_embeds.gpu[:num_tokens_padded] + elif getattr(getattr(self, "model", None), "has_preprocess", False): + input_ids = self.input_ids.gpu[:num_tokens_padded] + inputs_embeds = self.inputs_embeds.gpu[:num_tokens_padded] else: input_ids = self.input_ids.gpu[:num_tokens_padded] inputs_embeds = None + # ---------------------------------------Omni-new---------------------------------------------- if self.uses_mrope: positions = self.mrope_positions.gpu[:, :num_tokens_padded] @@ -290,6 +377,9 @@ class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner): aclgraph_runtime_mode=cudagraph_runtime_mode, batch_descriptor=batch_desc, model_instance=self.model, + has_sinks=self._has_sinks, + input_ids=input_ids, + eplb_heat_collection_status=self.eplb_heat_collection_status if self.dynamic_eplb else False, ): # ---------------------------------------Omni-new---------------------------------------------- if getattr(self.model, "talker", None) is not None and self.has_talker_mtp: @@ -309,6 +399,7 @@ class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner): positions=positions, intermediate_tensors=intermediate_tensors, inputs_embeds=inputs_embeds, + **model_kwargs, ) # ---------------------------------------Omni-new---------------------------------------------- if self.use_aux_hidden_state_outputs: @@ -320,7 +411,7 @@ class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner): # ---------------------------------------Omni-new---------------------------------------------- dummy_compute_logits(hidden_states) - if self.drafter: + if self.drafter and not profile_cpp: self.drafter.dummy_run( num_tokens=num_tokens_padded, with_prefill=with_prefill, @@ -333,10 +424,10 @@ class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner): is_profile=is_profile, ) if is_profile and self.dynamic_eplb: - target = self.model.language_model if hasattr(self.model, "language_model") else self.model - target.clear_all_moe_loads() - if self.dynamic_eplb: - self.eplb_updator.forward_end() + self.eplb_updator.adaptor.clear_all_moe_loads() + if not is_profile and self.dynamic_eplb: + self.eplb_updator.forward_end(self.eplb_heat_collection_status) + self._finalize_dump_data(dump=False) return hidden_states, hidden_states def _model_forward( @@ -375,11 +466,11 @@ class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner): if self.enable_enpu: # The soft segmentation scenario requires event.record first, then event.wait. - self._update_full_graph_params_if_needed(forward_context, num_tokens_padded, positions) + self._update_full_graph_params_if_needed(forward_context, num_tokens_padded) model_output = run_model() else: model_output = run_model() - self._update_full_graph_params_if_needed(forward_context, num_tokens_padded, positions) + self._update_full_graph_params_if_needed(forward_context, num_tokens_padded) # Omni-specific: wrap output if needed if not isinstance(model_output, OmniOutput) and hasattr(self.model, "make_omni_output"): From b8e7b21188ab9145b92de66d72e2edf5bab9ee36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zeyu=20Huang=20=7C=20=E9=BB=83=E6=BE=A4=E5=AE=87?= <11222265+fhfuih@users.noreply.github.com> Date: Wed, 29 Jul 2026 12:42:25 +0800 Subject: [PATCH 17/37] [skip ci][Misc] remove stray md introduced in #4652 (#5473) Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com> --- realtime_video_prompt_interaction_protocol.md | 138 ------------------ 1 file changed, 138 deletions(-) delete mode 100755 realtime_video_prompt_interaction_protocol.md diff --git a/realtime_video_prompt_interaction_protocol.md b/realtime_video_prompt_interaction_protocol.md deleted file mode 100755 index 534d76a51..000000000 --- a/realtime_video_prompt_interaction_protocol.md +++ /dev/null @@ -1,138 +0,0 @@ -# Realtime Video Prompt Interaction Protocol - -This document specifies the prompt-update subset of the realtime video -interaction protocol exposed by: - -```text -WebSocket /v1/realtime/video -``` - -A WebSocket connection contains exactly one video-generation request. Clients -therefore do not send a request or session ID with an interaction. The server -assigns a `request_id` in `video.start` and includes it in subsequent -server-to-client messages. - -## Interaction Request - -After `video.start`, a client may submit a prompt update: - -```json -{ - "type": "session.interaction", - "interaction": { - "event_id": "ui-update-42", - "event": { - "prompt": "A snowy street illuminated by warm lanterns" - }, - "transition_chunks": 3 - } -} -``` - -`interaction` has the following fields: - -- `event_id` (optional string): session-unique idempotency key. The - server preserves a client-provided value or generates a UUID when omitted. -- `event.prompt` (required string): non-empty target prompt. - over which the prompt reaches its target. -- `transition_chunks` (optional non-negative integer): model-generation chunks - over which the prompt reaches its target. If omitted, the model's default transition is used. - -Note: Although `["interaction"]["event"]` only contains `prompt`, still consider using `OmniTextPrompt` for this part (with total=false) so that other fields will be supported in the future. Currently, `vllm_omni/diffusion/worker/diffusion_model_runner.py` already takes care of enforcing subset format, routing to meaningful input handing routines, and raising `NotImplementedError`. Only need to adapt this design to the input data structure above. - -### Transition Semantics - -Prompt transitions are target updates and begin at the next eligible model -generation boundary after the interaction is processed. - -- `transition_chunks = 0` is a hard update: the first affected generation - chunk uses the target prompt. -- `transition_chunks = N`, where `N > 0`, reaches the target in the `N`th - affected generation chunk. For example, a three-chunk linear transition uses - progress values `1/3`, `2/3`, and `1`. - -### Idempotency and Replacement - -Within a session: - -- repeating an `event_id` with the same payload is a retry and must not - apply the update twice; -- repeating an `event_id` with a different payload is rejected with - `event_id_conflict`; -- a newer prompt target supersedes any older pending prompt target; -- when a new target interrupts an active transition, its transition starts - from the prompt state resolved at its effective boundary. - -## Queue Acknowledgement - -The server quickly acknowledges admission to the interaction queue: - -```json -{ - "type": "session.interaction.queued", - "request_id": "video_stream-...", - "event_id": "ui-update-42" -} -``` - -`queued` confirms syntax validation, idempotency handling, and queue admission. -It does not claim that the model has scheduled or applied the update. A later -failure is reported as a correlated error: - -```json -{ - "type": "error", - "request_id": "video_stream-...", - "event_id": "ui-update-42", - "code": "interaction_failed", - "message": "The active model does not support prompt updates" -} -``` - -## Video Chunk Metadata - -Each emitted binary payload is preceded by a JSON metadata message. The -metadata message and its binary payload form one ordered pair and must not be -interleaved with another outbound message. - -```json -{ - "type": "video.chunk_metadata", - "request_id": "video_stream-...", - "kind": "media", - "transport_chunk_index": 7, - "generation_chunk_index": 6, - "num_frames": 9, - "byte_length": 184320, - "started_event_ids": ["ui-update-42"], - "active_event_ids": ["ui-update-42"], - "completed_event_ids": [] -} -``` - -- `transport_chunk_index` identifies the binary WebSocket payload. -- `generation_chunk_index` identifies the model-generation chunk represented - by the payload. -- `started_event_ids` lists interactions that first affect this - generation chunk. -- `active_event_ids` lists interactions contributing to this generation - chunk, including an ongoing transition. -- `completed_event_ids` lists interactions that reach their target in - this generation chunk. - -Encoder trailer bytes are emitted as: - -```json -{ - "type": "video.chunk_metadata", - "request_id": "video_stream-...", - "kind": "trailer", - "transport_chunk_index": 8, - "generation_chunk_index": null, - "num_frames": 0, - "byte_length": 824, - "started_event_ids": [], - "active_event_ids": [], - "completed_event_ids": [] -} -``` From 1c6e7313394923000215a3299f4f79ede3873ecc Mon Sep 17 00:00:00 2001 From: BruceLoveDecimal <39156883+BruceLoveDecimal@users.noreply.github.com> Date: Wed, 29 Jul 2026 13:24:51 +0800 Subject: [PATCH 18/37] =?UTF-8?q?fix(diffusion):=20convert=20caller=20samp?= =?UTF-8?q?ling=20params=20instead=20of=20discarding=20=E2=80=A6=20(#4115)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: BruceLoveDecimal <39156883+BruceLoveDecimal@users.noreply.github.com> Co-authored-by: Gao Han --- .../inputs/test_diffusion_sampling_params.py | 65 +++++++++++++++++++ vllm_omni/engine/stage_pool.py | 11 ++-- vllm_omni/inputs/data.py | 25 +++++++ 3 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 tests/inputs/test_diffusion_sampling_params.py diff --git a/tests/inputs/test_diffusion_sampling_params.py b/tests/inputs/test_diffusion_sampling_params.py new file mode 100644 index 000000000..5daa1fd49 --- /dev/null +++ b/tests/inputs/test_diffusion_sampling_params.py @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Regression tests for diffusion sampling params coercion.""" + +from __future__ import annotations + +from typing import Any + +import pytest +from vllm.sampling_params import SamplingParams + +from vllm_omni.inputs.data import OmniDiffusionSamplingParams + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _sampling_params(*, seed: int | None = None, extra_args: dict[str, Any] | None = None) -> SamplingParams: + return SamplingParams(max_tokens=1, seed=seed, extra_args=extra_args) + + +def test_from_params_returns_existing_omni_params() -> None: + params = OmniDiffusionSamplingParams(num_inference_steps=20) + + converted = OmniDiffusionSamplingParams.from_params(params) + + assert converted is params + + +def test_from_params_converts_sampling_params_seed_and_known_extra_args() -> None: + params = _sampling_params( + seed=1234, + extra_args={ + "num_inference_steps": 50, + "height": 1024, + "width": 768, + "guidance_scale": 7.5, + }, + ) + + converted = OmniDiffusionSamplingParams.from_params(params) + + assert converted.seed == 1234 + assert converted.num_inference_steps == 50 + assert converted.height == 1024 + assert converted.width == 768 + assert converted.guidance_scale == 7.5 + assert converted.extra_args == {} + + +def test_from_params_preserves_unknown_extra_args() -> None: + params = _sampling_params( + extra_args={ + "height": 512, + "pipeline_specific": "kept", + }, + ) + + converted = OmniDiffusionSamplingParams.from_params(params) + + assert converted.height == 512 + assert converted.extra_args == {"pipeline_specific": "kept"} + + +def test_from_params_rejects_unsupported_types() -> None: + with pytest.raises(TypeError, match="Diffusion stage requires OmniDiffusionSamplingParams"): + OmniDiffusionSamplingParams.from_params({"height": 512}) diff --git a/vllm_omni/engine/stage_pool.py b/vllm_omni/engine/stage_pool.py index 487134db1..820b28510 100644 --- a/vllm_omni/engine/stage_pool.py +++ b/vllm_omni/engine/stage_pool.py @@ -937,10 +937,9 @@ class StagePool: ) -> int: """Submit a stage-entry request into this pool.""" params = params_override if params_override is not None else req_state.sampling_params_list[self.stage_id] - # Convert plain vllm SamplingParams for single-stage diffusion models - # that receive sampling params from the user/caller directly. - if self.stage_type == "diffusion" and not isinstance(params, OmniDiffusionSamplingParams): - params = OmniDiffusionSamplingParams() + # Direct engine callers may provide plain vLLM SamplingParams. + if self.stage_type == "diffusion": + params = OmniDiffusionSamplingParams.from_params(params) submit_kwargs = dict(submit_kwargs or {}) if self.stage_type == "diffusion": if isinstance(request, list): @@ -1003,8 +1002,8 @@ class StagePool: ) -> int: """Submit a streaming update to an already admitted request.""" params = req_state.sampling_params_list[self.stage_id] - if self.stage_type == "diffusion" and not isinstance(params, OmniDiffusionSamplingParams): - params = OmniDiffusionSamplingParams() + if self.stage_type == "diffusion": + params = OmniDiffusionSamplingParams.from_params(params) replica_id = self.get_bound_replica_id(request_id) if replica_id is None or self.clients[replica_id] is None: replica_id = await self._pick_or_select(request_id) diff --git a/vllm_omni/inputs/data.py b/vllm_omni/inputs/data.py index 6626176b6..61d3d12a3 100644 --- a/vllm_omni/inputs/data.py +++ b/vllm_omni/inputs/data.py @@ -380,5 +380,30 @@ class OmniDiffusionSamplingParams: def clone(self) -> "OmniDiffusionSamplingParams": return copy.deepcopy(self) + @classmethod + def from_params(cls, params: Any) -> "OmniDiffusionSamplingParams": + """Normalize caller params for diffusion stages. + + Existing Omni params are returned unchanged. Plain vLLM SamplingParams + copy seed and matching extra_args fields; remaining extra_args stay nested. + Unsupported types raise TypeError. + """ + if isinstance(params, cls): + return params + if isinstance(params, SamplingParams): + extra = dict(getattr(params, "extra_args", None) or {}) + known = set(cls.__dataclass_fields__) + mapped = {k: extra.pop(k) for k in list(extra) if k in known} + if extra: + mapped["extra_args"] = extra + seed = getattr(params, "seed", None) + if seed is not None: + mapped.setdefault("seed", seed) + return cls(**mapped) + raise TypeError( + "Diffusion stage requires OmniDiffusionSamplingParams or vllm.SamplingParams, " + f"got {type(params).__name__!r}." + ) + OmniSamplingParams: TypeAlias = SamplingParams | OmniDiffusionSamplingParams From 18be2ba3df93465cb77920c7e18bb45628be08e2 Mon Sep 17 00:00:00 2001 From: Canlin Guo Date: Wed, 29 Jul 2026 14:09:06 +0800 Subject: [PATCH 19/37] [Perf] Enable compile for MOSS-TTS-Local v1.5 talker and codec (#5530) Signed-off-by: Canlin Guo --- vllm_omni/deploy/moss_tts_local.yaml | 2 +- .../entrypoints/openai/serving_speech.py | 2 +- .../models/moss_tts/audio_tokenizer_v2.py | 109 +++++++++++++++--- .../cuda_graph_streaming_decoder_wrapper.py | 107 +++++++++++++++-- .../moss_tts/modeling_moss_tts_codec.py | 85 ++++++++++---- .../moss_tts/modeling_moss_tts_local_depth.py | 27 ++++- .../moss_tts/modeling_moss_tts_talker.py | 56 +++++++++ 7 files changed, 336 insertions(+), 52 deletions(-) diff --git a/vllm_omni/deploy/moss_tts_local.yaml b/vllm_omni/deploy/moss_tts_local.yaml index 7d3ae8601..2ed937e0a 100644 --- a/vllm_omni/deploy/moss_tts_local.yaml +++ b/vllm_omni/deploy/moss_tts_local.yaml @@ -54,7 +54,7 @@ stages: # torch_profiler_dir: /root/vllm-omni-workspace/vllm-omni/results/moss_local_stage1/batch_8 max_num_batched_tokens: 65536 max_model_len: 65536 - devices: "1" + devices: "0" input_connectors: from_stage_0: shm default_sampling_params: diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index 1ee2ce668..9cc6118e9 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -1764,7 +1764,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # headline improvement is multilingual synthesis when the language is # given (build_user_message(..., language=...)); 1.0 ignores it # gracefully. Sound-effect output is non-verbal, so skip it there. - if v in ("tts", "ttsd", "voice_generator") and getattr(request, "language", None): + if v in ("tts", "local", "ttsd", "voice_generator") and getattr(request, "language", None): user_kwargs["language"] = request.language # Build the unified-codes prompt: (L, 1+n_vq) where col 0 is text/special diff --git a/vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py b/vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py index 26ea9c567..098f53792 100644 --- a/vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py +++ b/vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py @@ -484,7 +484,6 @@ class RingKVCache: return slots = state_slot_ids.to(device=self.end_offset.device, dtype=torch.long) self.end_offset.index_fill_(0, slots, 0) - self.cache.index_fill_(1, slots, 0) def complete( self, @@ -1607,6 +1606,7 @@ class MossAudioTokenizerModel(MossAudioTokenizerPreTrainedModel): self._streaming_modules: list[StreamingModule] = [] self._streaming_exec_mask: torch.Tensor | None = None self._decoder_state_capacity = 0 + self._decoder_slot_offsets: torch.Tensor | None = None self.post_init() def _start_streaming(self, batch_size: int, *, decoder_only: bool = False) -> None: @@ -1634,6 +1634,49 @@ class MossAudioTokenizerModel(MossAudioTokenizerPreTrainedModel): root.apply(_start) self._streaming_modules = streaming_modules self._streaming_exec_mask = shared_exec_mask + if decoder_only: + self._centralize_decoder_slot_offsets(batch_size) + + def _centralize_decoder_slot_offsets(self, state_capacity: int) -> None: + """Pack per-module slot offsets so reset needs one CUDA kernel.""" + state_tensors: list[tuple[StreamingState, list[tuple[object, str]]]] = [] + for module in self._streaming_modules: + state = module._streaming_state + fields: list[tuple[object, str]] = [] + if isinstance(state, MHAState): + fields.append((state, "offset")) + if state.kv_cache is not None: + fields.append((state.kv_cache, "end_offset")) + elif isinstance(state, TransformerState): + fields.append((state, "offsets")) + else: + continue + + if not all( + isinstance(getattr(owner, name), torch.Tensor) + and getattr(owner, name).shape == (state_capacity,) + and getattr(owner, name).dtype == torch.long + for owner, name in fields + ): + raise RuntimeError( + "Dynamic decoder state slots require every per-slot offset to use the full decoder state capacity." + ) + state_tensors.append((state, fields)) + + num_offsets = sum(len(fields) for _, fields in state_tensors) + if num_offsets == 0: + raise RuntimeError("Dynamic decoder state pool has no per-slot offsets.") + storage = torch.zeros( + (num_offsets, state_capacity), + dtype=torch.long, + device=next(self.parameters()).device, + ) + row = 0 + for state, fields in state_tensors: + for owner, name in fields: + setattr(owner, name, storage[row]) + row += 1 + self._decoder_slot_offsets = storage def _stop_streaming(self) -> None: """Stop streaming mode for all modules.""" @@ -1642,6 +1685,7 @@ class MossAudioTokenizerModel(MossAudioTokenizerPreTrainedModel): self._streaming_modules = [] self._streaming_exec_mask = None self._decoder_state_capacity = 0 + self._decoder_slot_offsets = None def initialize_decoder_state_pool(self, state_capacity: int, scratch_capacity: int = 0) -> None: """Allocate persistent decoder state independently of execution B.""" @@ -1658,18 +1702,16 @@ class MossAudioTokenizerModel(MossAudioTokenizerPreTrainedModel): raise RuntimeError("MOSS Audio Tokenizer decoder state pool is not initialized.") if state_slot_ids.numel() == 0: return - slots = state_slot_ids.to(device=next(self.parameters()).device, dtype=torch.long) - if not torch.cuda.is_current_stream_capturing(): - slots_cpu = slots.detach().to("cpu") - if int(slots_cpu.min()) < 0 or int(slots_cpu.max()) >= self._decoder_state_capacity: + slot_offsets = self._decoder_slot_offsets + if slot_offsets is None: + raise RuntimeError("MOSS Audio Tokenizer decoder slot offsets are not initialized.") + if state_slot_ids.device.type == "cpu": + if int(state_slot_ids.min()) < 0 or int(state_slot_ids.max()) >= self._decoder_state_capacity: raise ValueError( - f"Decoder state slots must be in [0, {self._decoder_state_capacity}), got {slots_cpu.tolist()}" + f"Decoder state slots must be in [0, {self._decoder_state_capacity}), got {state_slot_ids.tolist()}" ) - for module in self._streaming_modules: - state = module._streaming_state - reset_slots = getattr(state, "reset_slots", None) - if callable(reset_slots): - reset_slots(slots) + slots = state_slot_ids.to(device=slot_offsets.device, dtype=torch.long) + slot_offsets.index_fill_(1, slots, 0) def decode_streaming_batch( self, @@ -1686,7 +1728,27 @@ class MossAudioTokenizerModel(MossAudioTokenizerPreTrainedModel): state_capacity=self._decoder_state_capacity, device=codes.device, ) - return self._decode_frame(codes, codes_lengths, execution_context=execution_context) + audio, audio_lengths = self.decode_streaming_tensors( + codes, + codes_lengths, + state_slot_ids, + valid_rows, + ) + return MossAudioTokenizerDecoderOutput(audio=audio, audio_lengths=audio_lengths) + + def decode_streaming_tensors( + self, + codes: torch.Tensor, + codes_lengths: torch.Tensor, + state_slot_ids: torch.Tensor, + valid_rows: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Tensor-only streaming decode boundary for vLLM compilation.""" + execution_context = StreamingExecutionContext( + state_slot_ids=state_slot_ids, + valid_rows=valid_rows, + ) + return self._decode_frame_tensors(codes, codes_lengths, execution_context=execution_context) def _set_streaming_exec_mask(self, exec_mask: torch.Tensor) -> None: """Update the shared active-slot mask with one device copy.""" @@ -1925,13 +1987,13 @@ class MossAudioTokenizerModel(MossAudioTokenizerPreTrainedModel): ) @torch.no_grad() - def _decode_frame( + def _decode_frame_tensors( self, codes: torch.Tensor, codes_lengths: torch.Tensor | None = None, execution_context: StreamingExecutionContext | None = None, - ) -> MossAudioTokenizerDecoderOutput: - """Detokenize discrete tokens into audio waveform.""" + ) -> tuple[torch.Tensor, torch.Tensor]: + """Detokenize codes and return tensors without Python output wrappers.""" nq, B, T = codes.shape device = codes.device @@ -1955,7 +2017,22 @@ class MossAudioTokenizerModel(MossAudioTokenizerPreTrainedModel): d, d_lengths = decoder_module(d, d_lengths) d, d_lengths = self._restore_channels_from_codec(d, d_lengths) - return MossAudioTokenizerDecoderOutput(audio=d, audio_lengths=d_lengths) + return d, d_lengths + + @torch.no_grad() + def _decode_frame( + self, + codes: torch.Tensor, + codes_lengths: torch.Tensor | None = None, + execution_context: StreamingExecutionContext | None = None, + ) -> MossAudioTokenizerDecoderOutput: + """Compatibility wrapper around the tensor-only decoder.""" + audio, audio_lengths = self._decode_frame_tensors( + codes, + codes_lengths, + execution_context=execution_context, + ) + return MossAudioTokenizerDecoderOutput(audio=audio, audio_lengths=audio_lengths) def encode( # type: ignore[override] self, diff --git a/vllm_omni/model_executor/models/moss_tts/cuda_graph_streaming_decoder_wrapper.py b/vllm_omni/model_executor/models/moss_tts/cuda_graph_streaming_decoder_wrapper.py index 4aeb17785..52815dd3b 100644 --- a/vllm_omni/model_executor/models/moss_tts/cuda_graph_streaming_decoder_wrapper.py +++ b/vllm_omni/model_executor/models/moss_tts/cuda_graph_streaming_decoder_wrapper.py @@ -5,18 +5,52 @@ from __future__ import annotations +import copy import time from dataclasses import dataclass import torch import torch.nn as nn from torch.cuda import CUDAGraph +from vllm.compilation.decorators import support_torch_compile +from vllm.config import CUDAGraphMode, VllmConfig +from vllm.config.vllm import set_current_vllm_config from vllm.logger import init_logger from vllm.platforms import current_platform logger = init_logger(__name__) +@support_torch_compile( + dynamic_arg_dims={ + "codes": {1: "batch", 2: "frames"}, + "codes_lengths": {0: "batch"}, + "state_slot_ids": {0: "batch"}, + "valid_rows": {0: "batch"}, + } +) +class _MossStreamingDecodeCompileAdapter(nn.Module): + """Expose the BF16 codec streaming hot path to vLLM compile. + + The dtype is part of the class identity intentionally: vLLM's AOT cache + drops runtime guards, so an artifact traced with the old FP32 decoder + weights must never be reused after the decoder is materialized as BF16. + """ + + def __init__(self, codec: nn.Module, *, vllm_config: VllmConfig) -> None: + super().__init__() + self.codec = codec + + def forward( + self, + codes: torch.Tensor, + codes_lengths: torch.Tensor, + state_slot_ids: torch.Tensor, + valid_rows: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + return self.codec.decode_streaming_tensors(codes, codes_lengths, state_slot_ids, valid_rows) + + @dataclass class _CapturedStreamingDecodeGraph: graph: CUDAGraph @@ -44,6 +78,7 @@ class CUDAGraphStreamingDecoderWrapper: batch_sizes: list[int], frame_sizes: list[int], num_quantizers: int, + vllm_config: VllmConfig, ) -> None: self.codec = codec self.state_capacity = int(state_capacity) @@ -53,6 +88,18 @@ class CUDAGraphStreamingDecoderWrapper: self.graphs: dict[tuple[int, int], _CapturedStreamingDecodeGraph] = {} self._pool = None self._warmed_up = False + # vLLM owns Inductor compilation; this wrapper remains the sole owner + # of CUDA Graph capture/replay because it understands persistent codec + # state slots. Disable vLLM's CUDA Graph layer to avoid nested graphs. + compile_config = copy.copy(vllm_config) + compile_config.compilation_config = copy.copy(vllm_config.compilation_config) + compile_config.compilation_config.cudagraph_mode = CUDAGraphMode.NONE + compile_config.compilation_config.static_forward_context = {} + with set_current_vllm_config(compile_config): + self._compiled_decode: nn.Module | None = _MossStreamingDecodeCompileAdapter( + codec, + vllm_config=compile_config, + ) @property def is_ready(self) -> bool: @@ -88,8 +135,12 @@ class CUDAGraphStreamingDecoderWrapper: for batch_size, frame_size in capture_keys: key = (batch_size, frame_size) try: - self._capture(batch_size, frame_size, device) - logger.info(" Captured MOSS-TTS streaming decoder CUDA graph for (B,T)=%s", key) + compiled = self._capture(batch_size, frame_size, device) + logger.info( + " Captured %s MOSS-TTS streaming decoder CUDA graph for (B,T)=%s", + "compiled" if compiled else "plain", + key, + ) except Exception: self.graphs.pop(key, None) logger.warning( @@ -105,7 +156,46 @@ class CUDAGraphStreamingDecoderWrapper: ) @torch.no_grad() - def _capture(self, batch_size: int, frame_size: int, device: torch.device) -> None: + def _capture(self, batch_size: int, frame_size: int, device: torch.device) -> bool: + if self._compiled_decode is not None: + try: + self._capture_with_decode( + batch_size, + frame_size, + device, + self._compiled_decode, + ) + return True + except Exception: + logger.warning( + "vLLM compile/capture failed for MOSS-TTS (B,T)=(%d,%d); falling back to a plain CUDA Graph", + batch_size, + frame_size, + exc_info=True, + ) + scratch_slots = self.state_capacity + torch.arange( + batch_size, + dtype=torch.long, + device=device, + ) + self.codec.reset_decoder_state_slots(scratch_slots) + self._compiled_decode = None + self._capture_with_decode( + batch_size, + frame_size, + device, + self.codec.decode_streaming_tensors, + ) + return False + + @torch.no_grad() + def _capture_with_decode( + self, + batch_size: int, + frame_size: int, + device: torch.device, + decode, + ) -> None: codes = torch.zeros( self.num_quantizers, batch_size, @@ -120,7 +210,8 @@ class CUDAGraphStreamingDecoderWrapper: stream = torch.cuda.Stream() stream.wait_stream(torch.cuda.current_stream()) with torch.cuda.stream(stream): - _ = self.codec.decode_streaming_batch(codes, lengths, scratch_slots, valid_rows) + for _ in range(3): + _ = decode(codes, lengths, scratch_slots, valid_rows) torch.cuda.current_stream().wait_stream(stream) torch.accelerator.synchronize(device) self.codec.reset_decoder_state_slots(scratch_slots) @@ -129,18 +220,16 @@ class CUDAGraphStreamingDecoderWrapper: self._pool = current_platform.get_global_graph_pool() graph = CUDAGraph() with torch.cuda.graph(graph, pool=self._pool, capture_error_mode="thread_local"): - output = self.codec.decode_streaming_batch(codes, lengths, scratch_slots, valid_rows) + audio, audio_lengths = decode(codes, lengths, scratch_slots, valid_rows) - if output.audio is None or output.audio_lengths is None: - raise RuntimeError("MOSS-TTS streaming decoder CUDA graph capture produced empty audio.") self.graphs[(batch_size, frame_size)] = _CapturedStreamingDecodeGraph( graph=graph, static_codes=codes, static_lengths=lengths, static_state_slot_ids=scratch_slots, static_valid_rows=valid_rows, - static_audio=output.audio, - static_audio_lengths=output.audio_lengths, + static_audio=audio, + static_audio_lengths=audio_lengths, ) def _select_batch_size(self, actual_batch_size: int) -> int | None: diff --git a/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py b/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py index d6bc525ac..ea45b9c02 100644 --- a/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py +++ b/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py @@ -14,6 +14,7 @@ from vllm.config import VllmConfig from vllm.logger import init_logger from vllm.model_executor.model_loader import DefaultModelLoader from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.utils.torch_utils import set_default_torch_dtype from vllm_omni.model_executor.models.moss_tts.audio_tokenizer import ( MossAudioTokenizerConfig, @@ -45,6 +46,7 @@ class _MossCodecStreamSession: *, state_capacity: int, n_vq: int, + vllm_config: VllmConfig, graph_batch_sizes: list[int] | None = None, graph_frame_sizes: list[int] | None = None, ) -> None: @@ -59,6 +61,13 @@ class _MossCodecStreamSession: batch_sizes = sorted({int(size) for size in (graph_batch_sizes or []) if 0 < int(size) <= self._state_capacity}) frame_sizes = sorted({int(size) for size in (graph_frame_sizes or []) if int(size) > 0}) scratch_capacity = max(batch_sizes, default=0) if self._device.type == "cuda" else 0 + self._total_state_capacity = self._state_capacity + scratch_capacity + self._state_slot_ids = torch.arange( + self._total_state_capacity, + device=self._device, + dtype=torch.long, + ) + self._samples_per_frame = int(codec.downsample_rate) initialize_state_pool = getattr(codec, "initialize_decoder_state_pool", None) if not callable(initialize_state_pool): raise RuntimeError("The streaming codec does not implement a decoder state pool.") @@ -71,6 +80,7 @@ class _MossCodecStreamSession: batch_sizes=batch_sizes, frame_sizes=frame_sizes, num_quantizers=self._n_vq, + vllm_config=vllm_config, ) self._cudagraph_wrapper.warmup(self._device) self.reset_slots(list(range(self._state_capacity + scratch_capacity))) @@ -84,24 +94,32 @@ class _MossCodecStreamSession: self._leased_slots.add(slot) return slot - def release(self, slot: int) -> None: + def release(self, slot: int, *, state_already_reset: bool = False) -> None: if self._closed: return if slot not in self._leased_slots: return - self.reset_slots([slot]) + if not state_already_reset: + self.reset_slots([slot]) self._leased_slots.remove(slot) self._free_stream_slots.append(slot) + def _reset_slot_ids(self, slot_ids: torch.Tensor) -> None: + reset_streaming_slots = getattr(self._codec, "reset_decoder_state_slots", None) + if not callable(reset_streaming_slots): + raise RuntimeError("The streaming codec does not implement state-slot reset.") + with torch.no_grad(): + reset_streaming_slots(slot_ids) + def reset_slots(self, slots: list[int]) -> None: if not slots: return - reset_streaming_slots = getattr(self._codec, "reset_decoder_state_slots", None) - if callable(reset_streaming_slots): - with torch.no_grad(): - reset_streaming_slots(torch.tensor(slots, device=self._device, dtype=torch.long)) - return - raise RuntimeError("The streaming codec does not implement state-slot reset.") + if min(slots) < 0 or max(slots) >= self._total_state_capacity: + raise ValueError(f"Decoder state slots must be in [0, {self._total_state_capacity}), got {slots}") + start = slots[0] + if slots != list(range(start, start + len(slots))): + raise ValueError(f"Decoder state slots must be contiguous, got {slots}") + self._reset_slot_ids(self._state_slot_ids[start : start + len(slots)]) def close(self) -> None: if self._closed: @@ -150,9 +168,8 @@ class _MossCodecStreamSession: used_cudagraph = graph_output is not None if used_cudagraph: - audio_tensor, lengths_tensor, actual_batch_size = graph_output + audio_tensor, _, actual_batch_size = graph_output audio_tensor = audio_tensor[:actual_batch_size] - lengths_tensor = lengths_tensor[:actual_batch_size] else: codes_lengths = torch.full( (len(slots),), @@ -170,16 +187,17 @@ class _MossCodecStreamSession: if result.audio is None: return {} audio_tensor = result.audio - lengths_tensor = result.audio_lengths + + if terminal_slots: + terminal_rows = [row for row, slot in enumerate(slots) if slot in terminal_slots] + terminal_slot_ids = state_slot_ids if len(terminal_rows) == len(slots) else state_slot_ids[terminal_rows] + self._reset_slot_ids(terminal_slot_ids) audio = audio_tensor.detach().to("cpu", torch.float32) - lengths = lengths_tensor.detach().to("cpu") if lengths_tensor is not None else None + audio_length = step_t * self._samples_per_frame out: dict[int, torch.Tensor] = {} for row, slot in enumerate(slots): - wav = audio[row] - if lengths is not None: - wav = wav[..., : int(lengths[row].item())] - out[slot] = wav.contiguous() + out[slot] = audio[row, ..., :audio_length].contiguous() return out @@ -479,6 +497,7 @@ class MossTTSCodecDecoder(nn.Module): self._codec, state_capacity=self._stream_state_capacity, n_vq=self._n_vq, + vllm_config=self.vllm_config, graph_batch_sizes=self._streaming_graph_batch_sizes, graph_frame_sizes=self._streaming_graph_frame_sizes, ) @@ -517,7 +536,12 @@ class MossTTSCodecDecoder(nn.Module): if wav is not None: outputs[output_index] = wav if finished: - self._finish_stream_request(request_id, session, slot) + self._finish_stream_request( + request_id, + session, + slot, + state_already_reset=True, + ) continue grouped.setdefault(int(codes_nq_t.shape[1]), []).append( @@ -533,7 +557,12 @@ class MossTTSCodecDecoder(nn.Module): if wav is not None: outputs[output_index] = wav if finished: - self._finish_stream_request(request_id, session, slot) + self._finish_stream_request( + request_id, + session, + slot, + state_already_reset=True, + ) return outputs @@ -568,9 +597,11 @@ class MossTTSCodecDecoder(nn.Module): request_id: str, session: _MossCodecStreamSession, slot: int | None, + *, + state_already_reset: bool = False, ) -> None: if slot is not None: - session.release(slot) + session.release(slot, state_already_reset=state_already_reset) self._stream_req_slots.pop(request_id, None) def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None: @@ -630,9 +661,17 @@ class MossTTSCodecDecoder(nn.Module): pass codec_path = self._codec_path - logger.info("Loading MOSS Audio Tokenizer from %s", codec_path) + device = self.vllm_config.device_config.device + logger.info("Loading MOSS Audio Tokenizer from %s directly on %s", codec_path, device) - codec_cfg, codec = self._build_codec(codec_path) + # This codec comes from a secondary checkpoint, so it cannot be built + # by the outer stage's normal initialize_model() call. Re-enter the + # same target-device/default-dtype contexts used by vLLM's + # BaseModelLoader.load_model() instead of constructing an 8 GiB FP32 + # model on CPU and copying the whole module to the GPU afterwards. + with set_default_torch_dtype(torch.float32): + with device: + codec_cfg, codec = self._build_codec(codec_path) model_loader = DefaultModelLoader(self.vllm_config.load_config) source = DefaultModelLoader.Source( @@ -710,9 +749,9 @@ class MossTTSCodecDecoder(nn.Module): skipped[:3] if skipped else "none", ) - device = self.vllm_config.device_config.device - codec.to(device=device, dtype=torch.float32) codec.eval() + if device.type != "cpu": + codec.decoder.to(dtype=torch.bfloat16) build_decode_lut = getattr(codec.quantizer, "build_decode_lut", None) if callable(build_decode_lut): lut_dtype = torch.bfloat16 if device.type == "cuda" else torch.float32 diff --git a/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local_depth.py b/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local_depth.py index e0d871b46..7ee551f29 100644 --- a/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local_depth.py +++ b/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_local_depth.py @@ -22,10 +22,14 @@ from __future__ import annotations import torch import torch.nn as nn import torch.nn.functional as F +from vllm.logger import init_logger from vllm_omni.model_executor.models.moss_tts.modeling_moss_tts_local import ( _sample_token, ) +from vllm_omni.platforms import current_omni_platform + +logger = init_logger(__name__) class _MossTTSLocalAttention(nn.Module): @@ -135,6 +139,7 @@ class MossTTSLocalDepthTransformer(nn.Module): rope_base = float(getattr(gpt2_config, "rope_base", 1_000_000.0)) self.h = nn.ModuleList([_MossTTSLocalBlock(self.hidden_size, n_head, inner_size, rope_base, eps)]) self.ln_f = nn.LayerNorm(self.hidden_size, eps=eps) + self._compiled_forward_prefix = None def _forward_prefix(self, seq_embeds: torch.Tensor) -> torch.Tensor: hidden_states = seq_embeds @@ -142,6 +147,24 @@ class MossTTSLocalDepthTransformer(nn.Module): hidden_states = block(hidden_states) return self.ln_f(hidden_states) + def setup_compile(self) -> None: + if self._compiled_forward_prefix is not None: + return + if not current_omni_platform.supports_torch_inductor(): + self._compiled_forward_prefix = self._forward_prefix + logger.warning_once("MOSS-TTS local depth torch.compile disabled on this platform") + return + self._compiled_forward_prefix = torch.compile( + self._forward_prefix, + dynamic=False, + options={"epilogue_fusion": False}, + ) + logger.info("MOSS-TTS local depth prefix enabled with torch.compile") + + def _run_prefix(self, seq_embeds: torch.Tensor) -> torch.Tensor: + forward_prefix = self._compiled_forward_prefix or self._forward_prefix + return forward_prefix(seq_embeds) + @torch.no_grad() def generate_frame( self, @@ -180,7 +203,7 @@ class MossTTSLocalDepthTransformer(nn.Module): embeds = backbone_last_hidden.new_zeros((batch_size, n_vq, self.hidden_size), dtype=dtype) embeds[:, 0, :] = backbone_last_hidden.to(dtype) - hidden = self._forward_prefix(embeds[:, :1, :]) + hidden = self._run_prefix(embeds[:, :1, :]) local_hidden = hidden[:, 0, :] binary_logits = local_text_lm_head(local_hidden).float() @@ -231,7 +254,7 @@ class MossTTSLocalDepthTransformer(nn.Module): if channel_index + 1 < n_vq: embeds[:, channel_index + 1, :] = audio_embeddings[channel_index](channel_token).to(dtype) - hidden = self._forward_prefix(embeds[:, : channel_index + 2, :]) + hidden = self._run_prefix(embeds[:, : channel_index + 2, :]) local_hidden = hidden[:, channel_index + 1, :] return should_continue, codes diff --git a/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py b/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py index 59313ecdf..171e57328 100644 --- a/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py +++ b/vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py @@ -1513,6 +1513,59 @@ class MossTTSLocalTalkerForGeneration(nn.Module): mtp_control[:, :1] = 1.0 if active else 0.0 return input_ids, text_embed, {"mtp_inputs": (mtp_hidden, mtp_control)} + def preprocess_decode_batch( + self, + *, + input_ids: torch.Tensor, + req_infos: list[dict[str, Any]], + ) -> tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + list[dict[str, Any]], + ]: + """Batch the tensor work from the decode branch of ``preprocess``.""" + input_ids = input_ids.reshape(-1) + batch_size = len(req_infos) + if input_ids.numel() != batch_size: + raise ValueError( + f"MOSS-TTS Local decode preprocess received {input_ids.numel()} token ids for {batch_size} requests" + ) + + text_embeds = self.model.embed_tokens(input_ids) + hidden_rows: list[torch.Tensor] = [] + active_rows: list[bool] = [] + for info in req_infos: + last_hidden = (info.get("hidden_states", {}) or {}).get("last") + if isinstance(last_hidden, torch.Tensor) and last_hidden.numel() > 0: + hidden_rows.append( + last_hidden.to( + device=text_embeds.device, + dtype=text_embeds.dtype, + ).reshape(1, -1) + ) + else: + hidden_rows.append(text_embeds.new_zeros((1, self.hidden_size))) + + state = info.get("audio_state", {}) or {} + active_rows.append(isinstance(state, dict) and not bool(state.get("is_stopping"))) + + last_talker_hidden = torch.cat(hidden_rows, dim=0) + text_step = torch.zeros_like(last_talker_hidden) + text_step[:, 0] = torch.tensor( + active_rows, + device=text_step.device, + dtype=text_step.dtype, + ) + return ( + input_ids, + text_embeds, + last_talker_hidden, + text_step, + [{} for _ in range(batch_size)], + ) + def postprocess(self, hidden_states: torch.Tensor, **_: Any) -> dict[str, Any]: if hidden_states.numel() == 0: return {} @@ -1681,6 +1734,9 @@ class MossTTSLocalTalkerForGeneration(nn.Module): [e.weight.detach() for e in self.audio_embeddings], dim=0 ) # (n_vq, audio_vocab_size, hidden_size) + if not self.vllm_config.model_config.enforce_eager: + self.local_transformer.setup_compile() + logger.info( "[MossTTSLocal] loaded %d/%d params; skipped=%d (first 5: %s)", len(loaded), From 7be7e0d249134cc121da7ed3945abd3d2473f14d Mon Sep 17 00:00:00 2001 From: R0CKSTAR Date: Wed, 29 Jul 2026 17:11:04 +0800 Subject: [PATCH 20/37] Support Python 3.10 runtime compatibility (#5366) Signed-off-by: Xiaodong Ye --- .../comfyui_vllm_omni/utils/types.py | 10 ++----- tests/comfyui/test_comfyui_integration.py | 20 ++++++------- .../entrypoints/test_stream_finish_reason.py | 10 ------- tests/model_tests/diffusion/config_types.py | 30 +++++++++---------- vllm_omni/inputs/data.py | 18 ++++------- vllm_omni/outputs/output_modality.py | 21 ++++--------- 6 files changed, 38 insertions(+), 71 deletions(-) diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py index c7d254eb9..cedd11068 100644 --- a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py @@ -1,12 +1,8 @@ from collections.abc import Callable from enum import Enum, auto -from typing import ( - Any, - Literal, - NotRequired, - TypeAlias, - TypedDict, -) +from typing import Any, Literal, TypeAlias + +from typing_extensions import NotRequired, TypedDict AudioFormat: TypeAlias = Literal["mp3", "opus", "aac", "flac", "wav", "pcm"] diff --git a/tests/comfyui/test_comfyui_integration.py b/tests/comfyui/test_comfyui_integration.py index f532c350f..e15098b4e 100644 --- a/tests/comfyui/test_comfyui_integration.py +++ b/tests/comfyui/test_comfyui_integration.py @@ -10,7 +10,7 @@ import multiprocessing import time import traceback from collections.abc import Iterable, Sequence -from enum import StrEnum, auto +from enum import Enum from types import SimpleNamespace from typing import Any, NamedTuple @@ -57,15 +57,15 @@ class SamplingCase(NamedTuple): lora: dict | None = None -class SamplingKind(StrEnum): - IMAGE_NONE = auto() - IMAGE_DIFFUSION_SINGLE = auto() - UNDERSTANDING_NONE = auto() - UNDERSTANDING_AR_LIST = auto() - TTS_NONE = auto() - TTS_DIFFUSION_SINGLE = auto() - VIDEO_NONE = auto() - VIDEO_DIFFUSION_SINGLE = auto() +class SamplingKind(str, Enum): + IMAGE_NONE = "image_none" + IMAGE_DIFFUSION_SINGLE = "image_diffusion_single" + UNDERSTANDING_NONE = "understanding_none" + UNDERSTANDING_AR_LIST = "understanding_ar_list" + TTS_NONE = "tts_none" + TTS_DIFFUSION_SINGLE = "tts_diffusion_single" + VIDEO_NONE = "video_none" + VIDEO_DIFFUSION_SINGLE = "video_diffusion_single" # Pre-defined arguments to be used in function calls during the tests diff --git a/tests/entrypoints/test_stream_finish_reason.py b/tests/entrypoints/test_stream_finish_reason.py index 63ec9f858..5ba3fea68 100644 --- a/tests/entrypoints/test_stream_finish_reason.py +++ b/tests/entrypoints/test_stream_finish_reason.py @@ -16,20 +16,10 @@ Key invariants tested: - voice/speaker parameter compatibility in chat completions """ -import enum import json from unittest.mock import MagicMock import pytest - -# Python 3.10 compat: StrEnum was added in 3.11 -if not hasattr(enum, "StrEnum"): - - class _StrEnum(str, enum.Enum): - """Minimal StrEnum backport for Python 3.10.""" - - enum.StrEnum = _StrEnum # type: ignore[attr-defined] - from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest, ChatCompletionResponseStreamChoice, diff --git a/tests/model_tests/diffusion/config_types.py b/tests/model_tests/diffusion/config_types.py index da599c2c2..49845fa65 100644 --- a/tests/model_tests/diffusion/config_types.py +++ b/tests/model_tests/diffusion/config_types.py @@ -3,7 +3,7 @@ Common definitions for controlling what tests run where. """ from collections.abc import Callable -from enum import StrEnum, auto +from enum import Enum from typing import NamedTuple, TypeAlias from pytest import MarkDecorator @@ -15,26 +15,26 @@ from vllm_omni.entrypoints.omni import Omni TinyDiffusionBuilder: TypeAlias = Callable[[], str] -class DiffusionAccs(StrEnum): +class DiffusionAccs(str, Enum): """Supported acceleration types / test settings for Diffusion Models.""" - HSDP = auto() - TEA_CACHE = auto() - CACHE_DIT = auto() - SEQUENCE_PARALLEL = auto() - CFG_PARALLEL = auto() - TENSOR_PARALLEL = auto() - CPU_OFFLOAD = auto() - LAYERWISE_OFFLOAD = auto() - VAE_PATCH_PARALLEL = auto() + HSDP = "hsdp" + TEA_CACHE = "tea_cache" + CACHE_DIT = "cache_dit" + SEQUENCE_PARALLEL = "sequence_parallel" + CFG_PARALLEL = "cfg_parallel" + TENSOR_PARALLEL = "tensor_parallel" + CPU_OFFLOAD = "cpu_offload" + LAYERWISE_OFFLOAD = "layerwise_offload" + VAE_PATCH_PARALLEL = "vae_patch_parallel" -class DiffusionTasks(StrEnum): +class DiffusionTasks(str, Enum): """Supported tasks for Diffusion Models.""" - TEXT_TO_IMAGE = auto() - IMAGE_TO_IMAGE = auto() - TEXT_TO_VIDEO = auto() + TEXT_TO_IMAGE = "text_to_image" + IMAGE_TO_IMAGE = "image_to_image" + TEXT_TO_VIDEO = "text_to_video" # Text to audio, etc should be added here as needed diff --git a/vllm_omni/inputs/data.py b/vllm_omni/inputs/data.py index 61d3d12a3..f0344c994 100644 --- a/vllm_omni/inputs/data.py +++ b/vllm_omni/inputs/data.py @@ -1,24 +1,16 @@ import copy import pprint from dataclasses import asdict, dataclass, field -from typing import Any, TypeAlias, TypedDict +from typing import Any, TypeAlias -from vllm.inputs import PromptType +import torch +from typing_extensions import NotRequired, TypedDict +from vllm.inputs import EmbedsPrompt, PromptType, TextPrompt, TokensPrompt +from vllm.inputs.engine import TokensInput from vllm.sampling_params import SamplingParams from vllm_omni.lora.request import LoRARequest -try: - from typing import NotRequired -except ImportError: - # Python < 3.11: use typing_extensions - from typing_extensions import NotRequired - - -import torch -from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt -from vllm.inputs.engine import TokensInput - class OmniTextPrompt(TextPrompt): """Text prompt with optional embeddings and stage payloads. diff --git a/vllm_omni/outputs/output_modality.py b/vllm_omni/outputs/output_modality.py index 743669454..7cdbc1f19 100644 --- a/vllm_omni/outputs/output_modality.py +++ b/vllm_omni/outputs/output_modality.py @@ -8,23 +8,8 @@ for type-safe multimodal output routing and tensor merging. from __future__ import annotations import re -import sys from enum import Enum, Flag, auto -if sys.version_info >= (3, 11): - from enum import StrEnum -else: - # TODO: Remove this Python 3.10 compatibility shim when Python 3.10 support is dropped. - class StrEnum(str, Enum): - """``enum.StrEnum`` for Python 3.10, which predates it. - - ``__str__`` is taken from ``str`` so ``str(member)`` yields the member - value rather than ``"ClassName.MEMBER"``, matching 3.11 semantics. - """ - - __str__ = str.__str__ - - _MODALITY_ALIASES: dict[str, str] = { "speech": "audio", "images": "image", @@ -38,13 +23,17 @@ _MODALITY_ALIASES: dict[str, str] = { } -class OutputModalityNames(StrEnum): +class OutputModalityNames(str, Enum): """Keys for output modalities. TODO: (Alex) Integrate this with the big-flag enum below + throughout the code for better type safety (currently only used for output processor). """ + # Keep the string conversion provided by ``enum.StrEnum`` while using the + # Python 3.10-compatible ``str, Enum`` form. + __str__ = str.__str__ + TEXT = "text" IMAGE = "image" AUDIO = "audio" From 925347332f8190d172dfa8ac0c307a5cfa2320cb Mon Sep 17 00:00:00 2001 From: zzh <67720593+BLANKETusers@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:43:45 +0800 Subject: [PATCH 21/37] Add vae-patch-parallel-size and vae-use-tiling for HunyuanImage Benchmark (#5502) Signed-off-by: zzh <943967662@qq.com> --- tests/dfx/perf/tests/test_hunyuan_image3_it2i.json | 1 + tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json | 2 ++ tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json | 2 ++ tests/dfx/perf/tests/test_hunyuan_image_tp4.json | 2 ++ 4 files changed, 7 insertions(+) diff --git a/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json index 1b6902cba..ab8a042e1 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json +++ b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json @@ -94,6 +94,7 @@ "vae_patch_parallel_size": 2, "enable_expert_parallel": true }, + "vae_use_tiling": true, "input_connectors": { "from_stage_0": "rdma_connector" }, diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json b/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json index 7a699f53a..da761616a 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json @@ -21,6 +21,8 @@ "deploy_config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml", "tensor-parallel-size": 2, "cfg-parallel-size": 2, + "vae-patch-parallel-size": 4, + "vae-use-tiling": true, "enable-diffusion-pipeline-profiler": true } }, diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json b/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json index 521febd2d..a719b3bbe 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json @@ -21,6 +21,8 @@ "deploy-config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml", "tensor-parallel-size": 2, "ulysses-degree": 2, + "vae-patch-parallel-size": 4, + "vae-use-tiling": true, "enable-diffusion-pipeline-profiler": true } }, diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp4.json b/tests/dfx/perf/tests/test_hunyuan_image_tp4.json index f2d05564b..5a71af5d7 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image_tp4.json +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp4.json @@ -20,6 +20,8 @@ "serve_args": { "deploy-config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml", "tensor-parallel-size": 4, + "vae-patch-parallel-size": 4, + "vae-use-tiling": true, "enable-diffusion-pipeline-profiler": true } }, From 7c17111ff54f93d054cbcb95edd66a7d54e89807 Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Wed, 29 Jul 2026 23:53:03 +0800 Subject: [PATCH 22/37] [Refactor][CI] Reorganize test tree, and move slow e2e to weekly (#4951) Signed-off-by: wangyu <410167048@qq.com> --- .buildkite/amd/test-amd-merge.yml | 2 +- .buildkite/amd/test-amd-ready.yml | 2 +- .buildkite/cuda/test-merge.yml | 72 +- .buildkite/cuda/test-nightly.yml | 6 + .buildkite/cuda/test-ready.yml | 119 +- docs/contributing/ci/test_system_overview.md | 76 +- docs/contributing/ci/test_writing_guide.md | 165 +- docs/design/feature/tensor_parallel.md | 2 +- examples/model_display_names.yml | 1 - recipes/JD/JoyAI-VL-Interaction.md | 2 +- recipes/NVIDIA/GR00T-N1.7.md | 9 +- .../hunyuan}/hunyuan_image_ref.png | Bin .../qwen_image_edit_2511_test1.png | Bin .../qwen_image_edit_2511_test2.png | Bin tests/{ => config}/test_config_factory.py | 51 +- tests/conftest.py | 1 + .../test_omni_scheduler_mixin_timeouts.py | 2 + tests/core/test_prefix_cache.py | 3 + .../{ => diffusion}/ar_diffusion/__init__.py | 0 .../ar_diffusion/test_capability_runner.py | 0 .../ar_diffusion/test_engine_routing.py | 2 + .../ar_diffusion/test_kv_cache.py | 3 + .../ar_diffusion/test_kv_orchestration.py | 1 + .../ar_diffusion/test_paged_attention.py | 26 +- .../ar_diffusion/test_pipeline_bridge.py | 3 + .../ar_diffusion/test_slot_mapping.py | 3 + .../attention/test_attention_config.py | 7 + .../diffusion/attention/test_attention_sp.py | 2 + tests/diffusion/attention/test_flash_attn.py | 3 + .../attention/test_piecewise_attn.py | 3 + tests/diffusion/attention/test_sage_attn3.py | 3 + tests/diffusion/attention/test_ulysses_uaa.py | 2 + .../distributed/test_cfg_parallel.py | 1334 ++++++++++------- tests/diffusion/distributed/test_comm.py | 609 ++++---- .../distributed/test_pipeline_parallel.py | 551 ++++--- ...parallelism.py => test_tensor_parallel.py} | 130 +- .../test_vae_decode_parallelism.py | 276 ++-- tests/diffusion/hooks/test_hook_registry.py | 3 + .../layers/test_rotary_emb_equivalence.py | 2 + .../models/dreamzero}/__init__.py | 0 .../models}/dreamzero/test_crossattn_cache.py | 0 .../models}/dreamzero/test_deploy_configs.py | 0 .../models}/dreamzero/test_dreamzero_utils.py | 0 .../dreamzero/test_fused_qk_rms_norm.py | 0 .../models}/dreamzero/test_pipeline_state.py | 0 .../models}/dreamzero/test_qkv_fusion.py | 0 .../test_session_state_accounting.py | 0 .../test_session_state_equivalence.py | 0 .../dreamzero/test_session_state_lifecycle.py | 0 .../models/ltx2/test_ltx2_transformer.py | 2 + .../sensenova_u1/test_sensenova_u1_cfg.py | 2 + .../offloader/test_diffusion_cpu_offload.py | 24 +- .../test_diffusion_layerwise_offload.py | 28 +- .../quantization}/test_gguf_diffusion.py | 5 +- .../quantization/test_quantization_quality.py | 2 + .../test_trajectory_similarity_tool.py | 2 + .../test_diffusion_config_fields.py | 2 +- .../test_diffusion_config_propagation.py | 0 tests/diffusion/test_diffusion_engine.py | 33 +- .../test_diffusion_engine_metrics.py | 5 + tests/docs/test_generate_examples.py | 3 + tests/dreamzero/openpi_client_helper.py | 307 ---- tests/dreamzero/test_openpi_client_helper.py | 46 - .../upstream/openpi_test_client_ar.py | 381 ----- .../upstream/test_openpi_e2e_source_parity.py | 358 ----- tests/e2e/accuracy/conftest.py | 225 --- tests/e2e/accuracy/helpers.py | 347 ++++- tests/e2e/accuracy/test_dreamzero.py | 63 + tests/e2e/accuracy/test_gebench_h100_smoke.py | 97 -- .../accuracy/test_gedit_bench_h100_smoke.py | 115 -- tests/e2e/accuracy/test_hunyuan_image3.py | 10 +- .../test_hunyuan_image3_pixel_accuracy.py | 3 +- tests/e2e/accuracy/test_qwen_image_edit.py | 6 +- .../upstream_socket_server_no_compile.py | 2 +- tests/{ => e2e/features}/comfyui/conftest.py | 2 +- .../comfyui/test_comfyui_integration.py | 4 +- .../test_async_omni_collective_rpc.py | 10 +- .../test_async_omni_qwen_image_generate.py | 8 +- .../custom_pipeline/test_worker_extension.py | 22 +- .../engine/test_duplex_control_client.py | 0 .../engine/test_duplex_control_plane.py | 0 .../engine/test_duplex_deploy_config.py | 0 .../engine/test_duplex_intermediate.py | 0 .../fullduplex/engine/test_duplex_lease.py | 0 .../fullduplex/engine/test_duplex_runtime.py | 0 .../fullduplex/minicpmo45/__init__.py | 0 .../minicpmo45/test_commit_policy.py | 0 .../fullduplex/minicpmo45/test_input.py | 0 .../fullduplex/openai/test_duplex_audio.py | 0 .../openai/test_runtime_adapter_boundary.py | 0 .../fullduplex/openai/test_websocket_actor.py | 0 .../features}/fullduplex/test_client.py | 0 .../features}/fullduplex/test_joyvl_brain.py | 0 .../features}/fullduplex/test_joyvl_policy.py | 0 .../features}/fullduplex/test_runtime.py | 0 .../helpers/custom_pipeline.py} | 288 +++- .../helpers/verl_omni_server.py} | 286 +--- .../features/rlhf_test/test_verl_omni_e2e.py | 217 +++ .../custom_pipeline/__init__.py | 14 - .../flow_match_sde_scheduler.py | 220 --- .../custom_pipeline/worker_extension.py | 77 - ...inger.py => test_soulxsinger_expansion.py} | 2 +- .../test_step_audio2_expansion.py | 4 +- .../test_dreamzero_expansion.py | 83 +- ...enpi.py => test_gr00t_openpi_expansion.py} | 78 +- .../test_omnivoice_expansion.py | 2 +- .../test_arg_utils_shared_fields.py} | 0 tests/{ => engine}/test_data_entry_keys.py | 0 tests/{ => engine}/test_omni_request.py | 0 .../test_omni_request_output.py} | 0 .../test_orchestrator_error_handling.py | 10 +- .../test_speaker_metadata_persistence.py | 0 .../openai_api/test_speech_usage.py | 2 + .../openai_api/test_stage_params.py | 3 + .../openai_api/test_tts_adapter.py | 3 + .../openai_api/test_video_frame_filter.py | 2 + .../openai_api/test_video_stream_handler.py | 3 + .../openai_api/test_video_stream_session.py | 3 + tests/entrypoints/test_pd_disaggregation.py | 3 + tests/entrypoints/test_stage_utils.py | 2 + tests/gr00t/__init__.py | 2 - tests/gr00t/openpi_client_helper.py | 183 --- tests/helpers/__init__.py | 9 +- tests/helpers/env.py | 27 + tests/helpers/runtime.py | 278 ++++ tests/helpers/{ => tests}/test_assertions.py | 2 + .../tests/test_media_pydub_removal.py} | 39 +- .../{ => tests}/test_runtime_stage_cli.py | 4 + .../cosyvoice3/test_cosyvoice3_components.py | 2 + .../dynin_omni/test_dynin_omni_token2audio.py | 18 + ...est_fish_speech_fast_ar_runtime_buffers.py | 3 + .../higgs_audio_v3/test_higgs_audio_v3.py | 44 +- .../test_mammoth_moda2_config.py | 3 - .../models/moss_tts/test_codec_cudagraph.py | 16 +- .../models/moss_tts/test_per_request_codes.py | 3 + .../qwen3_tts/test_cuda_graph_decoder.py | 24 +- .../test_qwen3_tts_talker_preprocess.py | 2 + .../test_tts_custom_voice_profiles.py | 0 ...st_runner_assisted_unified_graph_static.py | 5 + .../voxcpm2/test_talker_state_eviction.py | 3 + .../test_cosyvoice3_stage_input_processors.py | 3 + .../stage_input_processors/test_indextts2.py | 4 + .../test_qwen2_5_omni_thinker2talker.py | 2 + tests/plugin/__init__.py | 1 - tests/profile/test_omni_torch_profiler.py | 2 + tests/quantization/test_quack_fp8_gate.py | 3 + .../test_step_audio_reasoning_parser.py | 11 +- .../test_generate_nightly_perf_excel.py | 4 +- .../test_generate_nightly_perf_html.py | 4 +- tests/unit/higgs_audio_v3/__init__.py | 0 tests/utils/test_audio.py | 3 + tests/{ => utils}/test_speaker_cache.py | 0 .../test_speaker_cache_integration.py | 0 tests/utils/test_tracking_parser.py | 3 + tests/{ => utils}/test_version.py | 2 + .../nightly/buildkite_testcase_statistics.py | 4 +- .../deploy/step_audio2_thinker_ci.yaml | 2 +- vllm_omni/engine/arg_utils.py | 2 +- 158 files changed, 3479 insertions(+), 4130 deletions(-) rename tests/{e2e/accuracy/assets => assets/hunyuan}/hunyuan_image_ref.png (100%) rename tests/{e2e/accuracy/assets => assets/qwen_image_edit}/qwen_image_edit_2511_test1.png (100%) rename tests/{e2e/accuracy/assets => assets/qwen_image_edit}/qwen_image_edit_2511_test2.png (100%) rename tests/{ => config}/test_config_factory.py (97%) rename tests/{ => diffusion}/ar_diffusion/__init__.py (100%) rename tests/{ => diffusion}/ar_diffusion/test_capability_runner.py (100%) rename tests/{ => diffusion}/ar_diffusion/test_engine_routing.py (98%) rename tests/{ => diffusion}/ar_diffusion/test_kv_cache.py (99%) rename tests/{ => diffusion}/ar_diffusion/test_kv_orchestration.py (98%) rename tests/{ => diffusion}/ar_diffusion/test_paged_attention.py (94%) rename tests/{ => diffusion}/ar_diffusion/test_pipeline_bridge.py (99%) rename tests/{ => diffusion}/ar_diffusion/test_slot_mapping.py (97%) rename tests/diffusion/distributed/{test_zimage_parallelism.py => test_tensor_parallel.py} (53%) rename tests/{e2e/offline_inference/rlhf_test => diffusion/models/dreamzero}/__init__.py (100%) rename tests/{ => diffusion/models}/dreamzero/test_crossattn_cache.py (100%) rename tests/{ => diffusion/models}/dreamzero/test_deploy_configs.py (100%) rename tests/{ => diffusion/models}/dreamzero/test_dreamzero_utils.py (100%) rename tests/{ => diffusion/models}/dreamzero/test_fused_qk_rms_norm.py (100%) rename tests/{ => diffusion/models}/dreamzero/test_pipeline_state.py (100%) rename tests/{ => diffusion/models}/dreamzero/test_qkv_fusion.py (100%) rename tests/{ => diffusion/models}/dreamzero/test_session_state_accounting.py (100%) rename tests/{ => diffusion/models}/dreamzero/test_session_state_equivalence.py (100%) rename tests/{ => diffusion/models}/dreamzero/test_session_state_lifecycle.py (100%) rename tests/{plugin => diffusion/quantization}/test_gguf_diffusion.py (97%) rename tests/{ => diffusion}/test_diffusion_config_fields.py (97%) rename tests/{ => diffusion}/test_diffusion_config_propagation.py (100%) delete mode 100644 tests/dreamzero/openpi_client_helper.py delete mode 100644 tests/dreamzero/test_openpi_client_helper.py delete mode 100644 tests/dreamzero/upstream/openpi_test_client_ar.py delete mode 100644 tests/dreamzero/upstream/test_openpi_e2e_source_parity.py create mode 100644 tests/e2e/accuracy/test_dreamzero.py delete mode 100644 tests/e2e/accuracy/test_gebench_h100_smoke.py delete mode 100644 tests/e2e/accuracy/test_gedit_bench_h100_smoke.py rename tests/{dreamzero/upstream => e2e/accuracy/upstream_mock}/upstream_socket_server_no_compile.py (98%) rename tests/{ => e2e/features}/comfyui/conftest.py (97%) rename tests/{ => e2e/features}/comfyui/test_comfyui_integration.py (99%) rename tests/e2e/{offline_inference => features}/custom_pipeline/test_async_omni_collective_rpc.py (96%) rename tests/e2e/{offline_inference => features}/custom_pipeline/test_async_omni_qwen_image_generate.py (96%) rename tests/e2e/{offline_inference => features}/custom_pipeline/test_worker_extension.py (59%) rename tests/{ => e2e/features}/fullduplex/engine/test_duplex_control_client.py (100%) rename tests/{ => e2e/features}/fullduplex/engine/test_duplex_control_plane.py (100%) rename tests/{ => e2e/features}/fullduplex/engine/test_duplex_deploy_config.py (100%) rename tests/{ => e2e/features}/fullduplex/engine/test_duplex_intermediate.py (100%) rename tests/{ => e2e/features}/fullduplex/engine/test_duplex_lease.py (100%) rename tests/{ => e2e/features}/fullduplex/engine/test_duplex_runtime.py (100%) rename tests/{ => e2e/features}/fullduplex/minicpmo45/__init__.py (100%) rename tests/{ => e2e/features}/fullduplex/minicpmo45/test_commit_policy.py (100%) rename tests/{ => e2e/features}/fullduplex/minicpmo45/test_input.py (100%) rename tests/{ => e2e/features}/fullduplex/openai/test_duplex_audio.py (100%) rename tests/{ => e2e/features}/fullduplex/openai/test_runtime_adapter_boundary.py (100%) rename tests/{ => e2e/features}/fullduplex/openai/test_websocket_actor.py (100%) rename tests/{ => e2e/features}/fullduplex/test_client.py (100%) rename tests/{ => e2e/features}/fullduplex/test_joyvl_brain.py (100%) rename tests/{ => e2e/features}/fullduplex/test_joyvl_policy.py (100%) rename tests/{ => e2e/features}/fullduplex/test_runtime.py (100%) rename tests/e2e/{offline_inference/custom_pipeline/qwen_image_pipeline_with_logprob.py => features/helpers/custom_pipeline.py} (59%) rename tests/e2e/{offline_inference/rlhf_test/test_verl_omni_e2e.py => features/helpers/verl_omni_server.py} (52%) create mode 100644 tests/e2e/features/rlhf_test/test_verl_omni_e2e.py delete mode 100644 tests/e2e/offline_inference/custom_pipeline/__init__.py delete mode 100644 tests/e2e/offline_inference/custom_pipeline/flow_match_sde_scheduler.py delete mode 100644 tests/e2e/offline_inference/custom_pipeline/worker_extension.py rename tests/e2e/offline_inference/{test_soulxsinger.py => test_soulxsinger_expansion.py} (99%) rename tests/e2e/online_serving/{test_gr00t_openpi.py => test_gr00t_openpi_expansion.py} (66%) rename tests/{test_arg_utils.py => engine/test_arg_utils_shared_fields.py} (100%) rename tests/{ => engine}/test_data_entry_keys.py (100%) rename tests/{ => engine}/test_omni_request.py (100%) rename tests/{test_outputs.py => engine/test_omni_request_output.py} (100%) rename tests/{ => entrypoints/openai_api}/test_speaker_metadata_persistence.py (100%) delete mode 100644 tests/gr00t/__init__.py delete mode 100644 tests/gr00t/openpi_client_helper.py rename tests/helpers/{ => tests}/test_assertions.py (92%) rename tests/{test_pydub_removal.py => helpers/tests/test_media_pydub_removal.py} (85%) rename tests/helpers/{ => tests}/test_runtime_stage_cli.py (95%) rename tests/{unit => model_executor/models}/higgs_audio_v3/test_higgs_audio_v3.py (97%) rename tests/{unit => model_executor/models}/mammoth_moda2/test_mammoth_moda2_config.py (98%) rename tests/{ => model_executor/models/qwen3_tts}/test_tts_custom_voice_profiles.py (100%) delete mode 100644 tests/plugin/__init__.py rename tests/{ => tools}/test_generate_nightly_perf_excel.py (96%) rename tests/{ => tools}/test_generate_nightly_perf_html.py (95%) delete mode 100644 tests/unit/higgs_audio_v3/__init__.py rename tests/{ => utils}/test_speaker_cache.py (100%) rename tests/{ => utils}/test_speaker_cache_integration.py (100%) rename tests/{ => utils}/test_version.py (98%) rename tests/e2e/offline_inference/stage_configs/step_audio2_ci.yaml => vllm_omni/deploy/step_audio2_thinker_ci.yaml (93%) diff --git a/.buildkite/amd/test-amd-merge.yml b/.buildkite/amd/test-amd-merge.yml index 63433cc12..4e80d1968 100644 --- a/.buildkite/amd/test-amd-merge.yml +++ b/.buildkite/amd/test-amd-merge.yml @@ -89,7 +89,7 @@ steps: mirror_hardwares: [amdproduction] grade: Blocking commands: - - pytest -s -v tests/diffusion/distributed/test_zimage_parallelism.py + - pytest -s -v tests/diffusion/distributed/test_tensor_parallel.py - label: "Diffusion Model Test" agent_pool: mi325_1 diff --git a/.buildkite/amd/test-amd-ready.yml b/.buildkite/amd/test-amd-ready.yml index 8c645fff0..8f99ce909 100644 --- a/.buildkite/amd/test-amd-ready.yml +++ b/.buildkite/amd/test-amd-ready.yml @@ -35,7 +35,7 @@ steps: mirror_hardwares: [amdproduction] grade: Blocking commands: - - timeout 20m pytest -s -v tests/e2e/offline_inference/custom_pipeline/ -m "core_model" + - timeout 20m pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model" - label: "Voxtral TTS CUDA Unit Test" diff --git a/.buildkite/cuda/test-merge.yml b/.buildkite/cuda/test-merge.yml index d72b8f287..e971acb75 100644 --- a/.buildkite/cuda/test-merge.yml +++ b/.buildkite/cuda/test-merge.yml @@ -8,14 +8,24 @@ steps: - group: ":card_index_dividers: Simple Test" depends_on: upload-merge-pipeline steps: - - label: "Simple · Diffusion & Model Executor Test" + - label: "Simple · Model Executor Test" commands: - - "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + - "timeout 20m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + mirror_hardwares: l4_1 + + - label: "Simple · Diffusion Test" + commands: + - "timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + mirror_hardwares: l4_1 + + - label: "Simple · Engine&Entrypoints Test" + commands: + - "timeout 20m pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" mirror_hardwares: l4_1 - label: "Simple · Other Test" commands: - - "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + - "timeout 20m pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" mirror_hardwares: l4_1 - group: ":card_index_dividers: Diffusion Test" @@ -27,12 +37,6 @@ steps: - pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model" mirror_hardwares: l4_1 - - label: "Diffusion · Offloader Test" - timeout_in_minutes: 40 - commands: - - timeout 40m pytest -s -v tests/diffusion/offloader -m "advanced_model and cuda" - mirror_hardwares: l4_1 - - label: "Distributed Test" depends_on: upload-merge-pipeline timeout_in_minutes: 60 @@ -135,39 +139,7 @@ steps: timeout_in_minutes: 50 commands: - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'advanced_model and cuda' --run-level 'advanced_model' - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: "Omni · MiniCPM-o 4.5 Duplex Test" source_file_dependencies: @@ -210,22 +182,6 @@ steps: " mirror_hardwares: l4_1 - - label: "TTS · SoulX-Singer Test" - source_file_dependencies: - - tests/e2e/offline_inference/test_soulxsinger.py - - vllm_omni/diffusion/models/soulx_singer/ - - vllm_omni/deploy/soulxsinger_svc.yaml - - vllm_omni/deploy/soulxsinger_svs.yaml - timeout_in_minutes: 30 - commands: - - | - timeout 30m bash -c " - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v tests/e2e/offline_inference/test_soulxsinger.py -k 'svc or precomputed' -m 'advanced_model and cuda' --run-level 'advanced_model' - " - mirror_hardwares: l4_1 - - label: "Diffusion · Qwen Image Edit Test" source_file_dependencies: - tests/e2e/online_serving/test_qwen_image_edit.py diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index 72b31d0ce..377b78d1a 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -458,6 +458,11 @@ steps: depends_on: upload-nightly-pipeline if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "diffusion-x2iat-test" steps: + - group: ":full_moon: Diffusion · Distributed Test" + timeout_in_minutes: 30 + commands: + - timeout 30m pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda' --run-level "full_model" + mirror_hardwares: l4_4 - label: ":full_moon: Diffusion · Offloader Test" timeout_in_minutes: 40 @@ -473,6 +478,7 @@ steps: - label: ":full_moon: Diffusion · Quantization Test with L4" timeout_in_minutes: 60 commands: + - pip install "vllm-gguf-plugin>=0.0.3" - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model" mirror_hardwares: l4_1 diff --git a/.buildkite/cuda/test-ready.yml b/.buildkite/cuda/test-ready.yml index b0ea85900..68149036d 100644 --- a/.buildkite/cuda/test-ready.yml +++ b/.buildkite/cuda/test-ready.yml @@ -8,22 +8,26 @@ steps: - group: ":card_index_dividers: Simple Test" depends_on: upload-ready-pipeline steps: - - label: "Simple · Diffusion & Model Executor Test" + - label: "Simple · Model Executor Test" commands: - - "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + - "timeout 20m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + mirror_hardwares: l4_1 + + - label: "Simple · Diffusion Test" + commands: + - "timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + mirror_hardwares: l4_1 + + - label: "Simple · Engine&Entrypoints Test" + commands: + - "timeout 20m pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" mirror_hardwares: l4_1 - label: "Simple · Other Test" commands: - - "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + - "timeout 20m pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" mirror_hardwares: l4_1 - - label: "Custom Pipeline Test" - depends_on: upload-ready-pipeline - commands: - - timeout 20m pytest -s -v tests/e2e/offline_inference/custom_pipeline/ -m "core_model" - mirror_hardwares: l4_1 - - group: ":card_index_dividers: Diffusion Test" depends_on: upload-ready-pipeline steps: @@ -104,11 +108,34 @@ steps: - group: ":card_index_dividers: E2E Test" depends_on: upload-ready-pipeline steps: + - label: "Custom Pipeline Test" + source_file_dependencies: + - tests/e2e/features/custom_pipeline/ + - tests/e2e/features/helpers/custom_pipeline.py + - vllm_omni/diffusion/ + - vllm_omni/engine/ + - vllm_omni/entrypoints/ + - vllm_omni/inputs/ + - vllm_omni/outputs/ + commands: + - timeout 20m pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model" + agents: + queue: "gpu_1_queue" + plugins: + - docker#v5.2.0: + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + always-pull: true + propagate-environment: true + environment: + - "HF_HOME=/fsx/hf_cache" + volumes: + - "/fsx/hf_cache:/fsx/hf_cache" - label: "RLHF · VeRL-Omni E2E Test" source_file_dependencies: - - tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py - - tests/e2e/offline_inference/custom_pipeline/ + - tests/e2e/features/rlhf_test/test_verl_omni_e2e.py + - tests/e2e/features/helpers/verl_omni_server.py + - tests/e2e/features/helpers/custom_pipeline.py - vllm_omni/diffusion/ - vllm_omni/engine/ - vllm_omni/entrypoints/ @@ -117,7 +144,7 @@ steps: commands: - | timeout 20m bash -c " - pytest -s -v tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py + pytest -s -v tests/e2e/features/rlhf_test/test_verl_omni_e2e.py " mirror_hardwares: l4_1 @@ -143,34 +170,7 @@ steps: timeout 20m bash -c " pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model' " - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: "Omni · MiniCPM-o 4.5 Online Test" source_file_dependencies: @@ -181,34 +181,7 @@ steps: timeout 20m bash -c " pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model' " - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: "Omni · MiniCPM-o 4.5 Duplex Test" source_file_dependencies: @@ -371,13 +344,3 @@ steps: pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'core_model' --run-level 'core_model' " mirror_hardwares: h100_1 - - - label: "Diffusion · GGUF Plugin Test" - soft_fail: true - source_file_dependencies: - - tests/plugin/test_gguf_diffusion.py - - vllm_omni/diffusion/model_loader - commands: - - pip install "vllm-gguf-plugin>=0.0.3" - - timeout 30m pytest -sv tests/plugin/test_gguf_diffusion.py - mirror_hardwares: l4_1 diff --git a/docs/contributing/ci/test_system_overview.md b/docs/contributing/ci/test_system_overview.md index f1d4fb2c2..335ed5c53 100644 --- a/docs/contributing/ci/test_system_overview.md +++ b/docs/contributing/ci/test_system_overview.md @@ -56,7 +56,11 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari / core_model and cpu <15min - /tests/{component_name}/test_xxx + + tests/{component}/… mirroring vllm_omni/{component}/
    + (e.g. tests/diffusion/, tests/engine/, tests/entrypoints/)
    + Do not add new top-level tests/ dirs unrelated to a component. +     L1 & L2
    Section 1 L1&L2: Purpose, Test Content, Directory Location, Example @@ -68,16 +72,18 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari L2
    (E2E across models & GPU-required UT) Online (basic deployment scenarios):
    dummy, normal inference function (output format, stream), some instance startup UT High-priority models + online basic scenarios; request success, non-empty output, format match (no Whisper/accuracy) - High-priority features (using random lightweight models) + High-priority features and component tests that require launching instances High-priority interfaces (using random lightweight models) core_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion - Model tests:
    - /tests/e2e/online_serving/test_{model_name}.py
    - Feature tests:
    - /tests/{component_name}/test_xxx
    - Interface tests:
    - /tests/entrypoints/test_xxx + Model E2E:
    + tests/e2e/online_serving/test_{model}.py
    + tests/e2e/offline_inference/test_{model}.py
    + Feature integration:
    + tests/e2e/features/<feature>/
    + (e.g. fullduplex/, custom_pipeline/, rlhf_test/)
    + Component / interface Test (GPU):
    + tests/{component}/…, tests/entrypoints/… L1 & L2
    @@ -90,18 +96,19 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari L3
    (Important Perf & Integration & Accuracy) Online & Offline (multiple deployment scenarios):
    real model, normal inference function, normal accuracy High/medium-priority models + key online/offline scenarios; real weights, Whisper/similarity, preset voice gender, basic accuracy - Medium-priority features (using random lightweight models) + Medium-priority features and component tests that require launching instances Medium-priority interfaces (using random lightweight models) advanced_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion <30min - Model tests:
    - /tests/e2e/online_serving/test_{model_name}.py
    - /tests/e2e/offline_inference/test_{model_name}.py
    - Feature tests:
    - /tests/{component_name}/test_xxx
    - Interface tests:
    - /tests/entrypoints/test_xxx + Model E2E:
    + tests/e2e/online_serving/test_{model}.py
    + tests/e2e/offline_inference/test_{model}.py
    + tests/e2e/accuracy/
    + Feature integration:
    + tests/e2e/features/<feature>/
    + Component / interface Test:
    + tests/{component}/…, tests/entrypoints/… L3
    @@ -114,26 +121,23 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari L4
    (Perf & Integration & Accuracy) Online: full functional scenarios + performance test + doc test + accuracy test High-priority models: function, performance, accuracy, and doc testing
    Medium-priority models: function and doc testing - Low-priority features (using real weights) + Low-priority features and component tests that require launching instances Low-priority interfaces (using real weights) full_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion <3 hour - Model tests:
    - /tests/e2e/online_serving/test_{model_name}_expansion.py
    - Feature tests:
    - /tests/{component_name}/test_xxx
    - Interface tests:
    - /tests/entrypoints/test_xxx
    + Model E2E:
    + tests/e2e/online_serving/test_{model}_expansion.py
    + tests/e2e/offline_inference/test_{model}_expansion.py
    + tests/e2e/accuracy/test_{model}.py
    + Feature integration:
    + tests/e2e/features/<feature>/
    + Component / interface Test:
    + tests/{component}/…, tests/entrypoints/…
    Performance:
    - /tests/dfx/perf/tests/test_qwen3_omni_*.json (Omni), test_tts.json (TTS),
    - test_voxcpm2.json, test_higgs_audio_v3.json, and
    - /tests/dfx/perf/tests/test_{diffusion_model}_vllm_omni.json (Diffusion)
    - Doc Test:
    - tests/examples/online_serving/test_{model_name}.py
    - tests/examples/offline_inference/test_{model_name}.py
    - Accuracy Test:
    - /tests/e2e/accuracy/test_{model_name}.py + tests/dfx/perf/tests/
    + Doc examples:
    + tests/examples/online_serving/, tests/examples/offline_inference/ L4
    @@ -152,11 +156,9 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari Depends on reality Stability:
    - /tests/dfx/stability/tests/test_qwen3_omni.json
    - /tests/dfx/stability/tests/test_wan22.json
    + tests/dfx/stability/tests/
    Reliability:
    - tests/dfx/reliability/test_reliability_{model_key}.py
    - (e.g. test_reliability_qwen3_omni.py, test_reliability_wan22.py, test_reliability_hunyuan_image.py, test_reliability_voxcpm2.py) + tests/dfx/reliability/test_reliability_{model}.py L5
    @@ -168,7 +170,9 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari -For per-level test authoring (directories, markers, examples), see [Test Writing Guide](./test_writing_guide.md). +**Test Dir placement (summary):** component / unit under `tests/{component}/` mirroring `vllm_omni/`; model E2E under `tests/e2e/online_serving/`, `tests/e2e/offline_inference/`, and `tests/e2e/accuracy/`; feature integration under `tests/e2e/features//`; doc example tests under `tests/examples/online_serving/` and `tests/examples/offline_inference/`; performance under `tests/dfx/perf/`; stability under `tests/dfx/stability/`; reliability under `tests/dfx/reliability/`. Do **not** add new top-level directories under `tests/` that are unrelated to a `vllm_omni` component (or to the established `e2e` / `dfx` / `helpers` / `examples` / `buildkite` layout). + +For per-level test authoring (markers, examples), see [Test Writing Guide](./test_writing_guide.md). ## Common Specifications diff --git a/docs/contributing/ci/test_writing_guide.md b/docs/contributing/ci/test_writing_guide.md index 131d20b83..397f5e83c 100644 --- a/docs/contributing/ci/test_writing_guide.md +++ b/docs/contributing/ci/test_writing_guide.md @@ -4,137 +4,20 @@ This document describes how to author tests for each CI level (L1–L5). For the ## Test Directory Structure -Legend: `✅` = test exists, `⬜` = suggested to add. -``` -vllm_omni/ tests/ -├── config/ → ├── config/ -│ ├── model.py │ └── test_model.py ⬜ -│ └── lora.py │ └── test_lora.py ⬜ -│ -├── core/ → ├── core/ -│ └── sched/ │ └── sched/ -│ ├── omni_ar_scheduler.py │ ├── test_omni_ar_scheduler.py ⬜ -│ ├── omni_generation_scheduler.py │ ├── test_omni_generation_scheduler.py ⬜ -│ └── output.py │ └── test_output.py ✅ currently in entrypoints/test_omni_new_request_data.py (tests output.OmniNewRequestData) -│ -├── diffusion/ → ├── diffusion/ -│ ├── diffusion_engine.py │ ├── test_diffusion_engine.py ⬜ -│ ├── attention/ │ ├── attention/ -│ │ ├── layer.py │ │ ├── test_attention_sp.py ✅ -│ │ └── backends/ │ │ └── test_flash_attn.py ✅ -│ ├── distributed/ │ ├── distributed/ -│ │ └── ... │ │ ├── test_comm.py ✅ -│ │ │ │ ├── test_cfg_parallel.py ✅ -│ │ │ │ └── test_sp_plan_hooks.py ✅ -│ ├── lora/ │ ├── lora/ -│ │ └── ... │ │ ├── test_base_linear.py ✅ -│ │ │ │ └── test_lora_manager.py ✅ -│ ├── models/ │ ├── models/ -│ │ ├── qwen_image/ │ │ ├── qwen_image/ (e2e coverage) -│ │ ├── ovis_image/ │ │ ├── ovis_image/ -│ │ │ └── ... │ │ │ └── test_ovis_image.py ✅ -│ │ ├── z_image/ │ │ └── z_image/ -│ │ └── ... │ │ └── test_zimage_tp_constraints.py ✅ -│ └── worker/ │ └── worker/ -│ ├── diffusion_worker.py │ └── test_diffusion_worker.py ✅ file at diffusion/test_diffusion_worker.py -│ └── diffusion_model_runner.py │ -│ -├── distributed/ → ├── distributed/ -│ └── omni_connectors/ │ └── omni_connectors/ -│ ├── adapter.py │ ├── test_adapter_and_flow.py ✅ -│ ├── kv_transfer_manager.py │ ├── test_basic_connectors.py ✅ -│ ├── connectors/ │ ├── test_kv_flow.py ✅ -│ └── utils/ │ └── test_omni_connector_configs.py ✅ -│ -├── engine/ → ├── engine/ -│ ├── input_processor.py │ ├── test_input_processor.py ⬜ (no processor.py in source) -│ ├── output_processor.py │ └── test_output_processor.py ⬜ -│ └── arg_utils.py │ └── test_arg_utils.py ⬜ -│ -├── entrypoints/ → ├── entrypoints/ -│ ├── stage_utils.py │ ├── test_stage_utils.py ✅ -│ ├── cli/ │ ├── cli/ (benchmarks/test_serve_cli.py covers CLI serve) -│ │ └── ... │ │ └── test_*.py ⬜ -│ └── openai/ │ └── openai_api/ # maps to entrypoints/openai/ -│ ├── api_server.py │ ├── test_api_server.py ⬜ (e2e indirect coverage) -│ ├── serving_chat.py │ ├── test_serving_chat_sampling_params.py ✅ -│ ├── serving_speech.py │ ├── test_serving_speech.py ✅ -│ └── image_api_utils.py │ └── test_image_server.py ✅ -│ -├── inputs/ → ├── inputs/ -│ ├── data.py │ ├── test_data.py ⬜ -│ ├── parse.py │ ├── test_parse.py ⬜ -│ └── preprocess.py │ └── test_preprocess.py ✅ currently in entrypoints/test_omni_input_preprocessor.py -│ -├── model_executor/ → ├── model_executor/ -│ ├── layers/ │ ├── layers/ -│ │ └── mrope.py │ │ └── test_mrope.py ⬜ -│ ├── model_loader/ │ ├── model_loader/ -│ │ └── weight_utils.py │ │ └── test_weight_utils.py ⬜ -│ ├── models/ │ ├── models/ -│ │ ├── qwen2_5_omni/ │ │ ├── qwen2_5_omni/ -│ │ │ ├── qwen2_5_omni_thinker.py │ │ │ ├── test_audio_length.py ✅ -│ │ │ ├── qwen2_5_omni_talker.py │ │ │ ├── test_qwen2_5_omni_thinker.py ⬜ -│ │ │ └── qwen2_5_omni_token2wav.py │ │ │ ├── test_qwen2_5_omni_talker.py ⬜ -│ │ └── qwen3_omni/ │ │ │ └── test_qwen2_5_omni_token2wav.py ⬜ -│ │ └── ... │ │ └── qwen3_omni/ -│ ├── stage_configs/ │ │ └── test_*.py ⬜ -│ │ └── *.yaml │ └── stage_configs/ (used by e2e, test_*.py can be added) ⬜ -│ └── stage_input_processors/ │ └── stage_input_processors/ -│ └── ... │ └── test_*.py ⬜ -│ -├── sample/ → ├── sample/ -│ └── __init__.py │ └── test_*.py ⬜ -│ -├── utils/ → ├── utils/ -│ └── __init__.py │ └── test_*.py ⬜ (no platform_utils.py currently) -│ -├── worker/ → ├── worker/ -│ ├── gpu_ar_model_runner.py │ ├── test_gpu_ar_model_runner.py ⬜ -│ ├── gpu_ar_worker.py │ ├── test_gpu_ar_worker.py ⬜ -│ ├── gpu_generation_model_runner.py │ ├── test_gpu_generation_model_runner.py ✅ -│ ├── gpu_generation_worker.py │ ├── test_gpu_generation_worker.py ⬜ -│ ├── gpu_model_runner.py │ ├── test_omni_gpu_model_runner.py ✅ -│ └── mixins.py │ └── (npu under platforms/npu/worker/) # not worker/npu/ -│ -├── platforms/ → (no tests/platforms/, e2e and stage_configs provide indirect coverage) -│ ├── cuda/ -│ ├── npu/worker/ # NPU worker here, not vllm_omni/worker/npu/ -│ ├── rocm/ -│ └── xpu/worker/ -│ -├── outputs.py → test_outputs.py ✅ (at tests root) -├── (logger, patch, request, version) → (no corresponding unit test) -│ -└── e2e (tests side only) → ├── e2e/ - ├── online_serving/ ✅ non-empty - │ ├── test_qwen2_5_omni.py - │ ├── test_async_omni.py - │ ├── test_qwen3_omni.py - │ ├── test_qwen3_omni_expansion.py - │ ├── test_mimo_audio.py - │ └── test_images_generations_lora.py - └── offline_inference/ ✅ - ├── test_qwen2_5_omni.py - ├── test_qwen3_omni.py - ├── test_bagel_text2img.py - ├── test_z_image.py - ├── test_wan22.py - ├── test_zimage_tensor_parallel.py - ├── test_cache_dit.py - ├── test_teacache.py - ├── test_stable_audio_expansion.py - ├── test_diffusion_cpu_offload.py - ├── test_diffusion_layerwise_offload.py - ├── test_diffusion_lora.py - ├── test_sequence_parallel.py - └── stage_configs/ (legacy schema, still - ├── bagel_*.yaml present for unmigrated - └── npu/, rocm/, etc. models) +Canonical placement by CI level is in the **Test Dir** column of the level matrix in [Test System Overview](./test_system_overview.md). -# Migrated models (qwen3_omni_moe, qwen2_5_omni, qwen3_tts) live under -# vllm_omni/deploy/ instead — see docs/configuration/stage_configs.md. -``` +Summary: + +- **Component / unit:** `tests/{component}/…` mirroring `vllm_omni/{component}/` +- **Model E2E:** `tests/e2e/online_serving/`, `tests/e2e/offline_inference/`, `tests/e2e/accuracy/` +- **Feature integration:** `tests/e2e/features//` +- **Doc example tests:** `tests/examples/online_serving/`, `tests/examples/offline_inference/` +- **Performance:** `tests/dfx/perf/tests/` (JSON configs; runners under `tests/dfx/perf/scripts/`) +- **Stability:** `tests/dfx/stability/tests/` +- **Reliability:** `tests/dfx/reliability/` +- Do **not** add new top-level `tests/` directories unrelated to a component (or to `e2e` / `dfx` / `helpers` / `examples` / `buildkite`) + +Deploy YAMLs: `vllm_omni/deploy/` via `tests.helpers.stage_config.get_deploy_config_path` (see [stage configs](../../configuration/stage_configs.md)). ## Markers for Tests @@ -373,13 +256,13 @@ L2 level testing builds upon L1 by introducing GPU resources and verifying that #### 1.3 Test Directory and Execution Files -A clear directory structure is key to managing test cases efficiently. +A clear directory structure is key to managing test cases efficiently. See the **Test Dir** column in [Test System Overview](./test_system_overview.md) for the canonical placement rules. -- ***L1 Test Directory***: `/tests/{component_name}/test_xxx.py` -- - Here, `{component_name}` corresponds to modules in the source code, such as `distributed`, `entrypoints`, etc., and `test_xxx.py` is the specific test file. -- ***L2 Test Directory***: -- - Online Serving: `/tests/e2e/online_serving/test_{model_name}.py` - - Offline Inference: `/tests/e2e/offline_inference/test_{model_name}.py` +- ***L1 (component / unit)***: `tests/{component_name}/…` mirroring `vllm_omni/{component_name}/` (e.g. `tests/diffusion/`, `tests/entrypoints/`). Do **not** invent new top-level `tests/` folders unrelated to a component. +- ***L2 (basic model E2E)***: + - Online serving: `tests/e2e/online_serving/test_{model_name}.py` + - Offline inference: `tests/e2e/offline_inference/test_{model_name}.py` +- ***Feature-level integration*** (cross-cutting, not a single-model smoke): `tests/e2e/features//` (e.g. `fullduplex/`, `custom_pipeline/`, `rlhf_test/`). #### 1.4 Execution Method and Example @@ -485,10 +368,12 @@ L3 level testing executes after code is merged into the main branch. Its core pu #### 2.3 Test Directory and Execution Files -- ***Functional Testing***: -- - Online Serving: `/tests/e2e/online_serving/test_{model_name}_expansion.py` - - Offline Inference: `/tests/e2e/offline_inference/test_{model_name}_expansion.py` - - (Note: `_expansion.py` likely means it contains more comprehensive scenario cases compared to L2 tests). +- ***Functional / model E2E***: + - Online serving: `tests/e2e/online_serving/test_{model_name}_expansion.py` + - Offline inference: `tests/e2e/offline_inference/test_{model_name}_expansion.py` + - Accuracy / similarity: `tests/e2e/accuracy/` + - (`_expansion.py` holds broader scenario coverage than the L2 smoke file.) +- ***Feature-level integration***: `tests/e2e/features//` (not under a new top-level `tests/` directory). #### 2.4 Execution Method and Example diff --git a/docs/design/feature/tensor_parallel.md b/docs/design/feature/tensor_parallel.md index bcafde7e7..a01e2a0e6 100644 --- a/docs/design/feature/tensor_parallel.md +++ b/docs/design/feature/tensor_parallel.md @@ -264,7 +264,7 @@ Complete examples in the codebase: | **Z-Image** | `vllm_omni/diffusion/models/z_image/z_image_transformer.py` | Standard TP | Full implementation with validation | | **FLUX** | `vllm_omni/diffusion/models/flux/flux_transformer.py` | Dual-stream | Image + text streams | | **Qwen-Image** | `vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py` | Standard TP | With RoPE | -| **TP Tests** | `tests/e2e/offline_inference/test_zimage_parallelism.py` | E2E testing | TP correctness and performance | +| **TP Tests** | `tests/diffusion/distributed/test_tensor_parallel.py` | E2E testing | TP correctness and performance | | **Constraint Tests** | `tests/diffusion/models/z_image/test_zimage_tp_constraints.py` | Unit testing | Validation logic | --- diff --git a/examples/model_display_names.yml b/examples/model_display_names.yml index be3f8e381..7a04ff8fa 100644 --- a/examples/model_display_names.yml +++ b/examples/model_display_names.yml @@ -8,5 +8,4 @@ mimo_audio: MiMo-Audio ming_flash_omni: Ming-flash-omni 2.0 qwen2_5_omni: Qwen2.5-Omni qwen3_omni: Qwen3-Omni -sensenova_u1: SenseNova-U1 step_audio2: Step-Audio2 diff --git a/recipes/JD/JoyAI-VL-Interaction.md b/recipes/JD/JoyAI-VL-Interaction.md index ed767830f..e09d5b16c 100644 --- a/recipes/JD/JoyAI-VL-Interaction.md +++ b/recipes/JD/JoyAI-VL-Interaction.md @@ -168,7 +168,7 @@ the model + orchestrator + WebUI together — point `WEBUI_DIR` at your clone. python examples/online_serving/joyvl_interaction/cli/run_cli_demo.py \ path/to/video.mp4 --query "Alert me if a fire breaks out" -pytest tests/fullduplex # framework + JoyVL unit tests +pytest tests/e2e/features/fullduplex # framework + JoyVL unit tests ``` ## Testing with an RTSP stream (optional) diff --git a/recipes/NVIDIA/GR00T-N1.7.md b/recipes/NVIDIA/GR00T-N1.7.md index 91f676a52..0a5270222 100644 --- a/recipes/NVIDIA/GR00T-N1.7.md +++ b/recipes/NVIDIA/GR00T-N1.7.md @@ -69,8 +69,13 @@ Notes: ## Verification ```python -from tests.gr00t.openpi_client_helper import run_policy_session, validate_session_result -validate_session_result(run_policy_session(host="127.0.0.1", port=8000)) +from tests.helpers.runtime import OpenAIClientHandler + +handler = OpenAIClientHandler(host="127.0.0.1", port=8000, log_stats=False) +response = handler.send_robot_openpi_ws_request( + {"run_default_policy_session": True, "session_id": "gr00t-smoke"} +)[0] +assert response.operation_responses[-1]["status"] == "reset successful" ``` Or run the e2e test suite: diff --git a/tests/e2e/accuracy/assets/hunyuan_image_ref.png b/tests/assets/hunyuan/hunyuan_image_ref.png similarity index 100% rename from tests/e2e/accuracy/assets/hunyuan_image_ref.png rename to tests/assets/hunyuan/hunyuan_image_ref.png diff --git a/tests/e2e/accuracy/assets/qwen_image_edit_2511_test1.png b/tests/assets/qwen_image_edit/qwen_image_edit_2511_test1.png similarity index 100% rename from tests/e2e/accuracy/assets/qwen_image_edit_2511_test1.png rename to tests/assets/qwen_image_edit/qwen_image_edit_2511_test1.png diff --git a/tests/e2e/accuracy/assets/qwen_image_edit_2511_test2.png b/tests/assets/qwen_image_edit/qwen_image_edit_2511_test2.png similarity index 100% rename from tests/e2e/accuracy/assets/qwen_image_edit_2511_test2.png rename to tests/assets/qwen_image_edit/qwen_image_edit_2511_test2.png diff --git a/tests/test_config_factory.py b/tests/config/test_config_factory.py similarity index 97% rename from tests/test_config_factory.py rename to tests/config/test_config_factory.py index 61d870e98..5a59dcf32 100644 --- a/tests/test_config_factory.py +++ b/tests/config/test_config_factory.py @@ -1145,7 +1145,7 @@ class TestResolveScheduler: class TestDeployConfigLoading: def test_load_minicpmo_duplex_deploy_config(self): - deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "minicpmo_4_5_duplex.yaml" + deploy_path = Path(get_deploy_config_path("minicpmo_4_5_duplex.yaml")) deploy = load_deploy_config(deploy_path) pipeline = resolve_pipeline_config("minicpmo_4_5") @@ -1182,7 +1182,7 @@ class TestDeployConfigLoading: stage2_devices: str, stage1_replicas: int, ): - deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / filename + deploy_path = Path(get_deploy_config_path(filename)) deploy = load_deploy_config(deploy_path) pipeline = resolve_pipeline_config("minicpmo_4_5") @@ -1226,7 +1226,7 @@ stages: assert {s.yaml_engine_args.get("custom_voice_dir") for s in stages} == {str(custom_voice_dir)} def test_load_qwen3_omni_moe_deploy_config(self): - deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_omni_moe.yaml" + deploy_path = Path(get_deploy_config_path("qwen3_omni_moe.yaml")) deploy = load_deploy_config(deploy_path) assert len(deploy.stages) == 3 assert deploy.async_chunk is True @@ -1234,7 +1234,7 @@ stages: assert deploy.platforms is not None def test_load_voxtral_tts_deploy_config_schema_fields(self): - deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "voxtral_tts.yaml" + deploy_path = Path(get_deploy_config_path("voxtral_tts.yaml")) deploy = load_deploy_config(deploy_path) assert deploy.stages[0].config_format == "mistral" assert deploy.stages[0].load_format == "mistral" @@ -1244,7 +1244,7 @@ stages: ) def test_load_ming_flash_omni_deploy_config_schema_fields(self): - deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "ming_flash_omni.yaml" + deploy_path = Path(get_deploy_config_path("ming_flash_omni.yaml")) deploy = load_deploy_config(deploy_path) assert deploy.stages[0].compilation_config == {"pass_config": {"fuse_allreduce_rms": False}} assert "compilation_config" not in deploy.stages[0].engine_extras @@ -1384,7 +1384,7 @@ stages: assert not list(stage_config_dir.glob("*.yaml")) def test_merge_pipeline_deploy(self): - deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_omni_moe.yaml" + deploy_path = Path(get_deploy_config_path("qwen3_omni_moe.yaml")) if not deploy_path.exists(): pytest.skip("Deploy config not found") @@ -1411,7 +1411,7 @@ stages: ) assert isinstance(pipeline, PipelineConfig) - base = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_omni_moe.yaml" + base = Path(get_deploy_config_path("qwen3_omni_moe.yaml")) if not base.exists(): pytest.skip("Deploy config not found") @@ -1831,7 +1831,7 @@ class TestQwen3TTSPipeline: def test_per_stage_model_arch_flows_through_merge(self, tmp_path): """Verify the new ps.model_arch override survives merge_pipeline_deploy.""" - deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_tts.yaml" + deploy_path = Path(get_deploy_config_path("qwen3_tts.yaml")) if not deploy_path.exists(): pytest.skip("qwen3_tts deploy yaml not found") @@ -1955,7 +1955,7 @@ class TestMingFlashOmniPipeline: def test_full_yaml_loads_and_merges(self): """deploy/ming_flash_omni.yaml parses and merges with the registered pipeline.""" - deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "ming_flash_omni.yaml" + deploy_path = Path(get_deploy_config_path("ming_flash_omni.yaml")) if not deploy_path.exists(): pytest.skip("ming_flash_omni deploy yaml not found") @@ -1975,7 +1975,7 @@ class TestMingFlashOmniPipeline: def test_tts_yaml_loads_and_merges(self): """deploy/ming_flash_omni_tts.yaml parses and routes to the TTS-only pipeline.""" - deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "ming_flash_omni_tts.yaml" + deploy_path = Path(get_deploy_config_path("ming_flash_omni_tts.yaml")) if not deploy_path.exists(): pytest.skip("ming_flash_omni_tts deploy yaml not found") @@ -2013,7 +2013,7 @@ class TestMingFlashOmniPipeline: def test_thinker_only_yaml_loads_and_merges(self): """deploy/ming_flash_omni_thinker_only.yaml parses and routes to the thinker-only pipeline.""" - deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "ming_flash_omni_thinker_only.yaml" + deploy_path = Path(get_deploy_config_path("ming_flash_omni_thinker_only.yaml")) if not deploy_path.exists(): pytest.skip("ming_flash_omni_thinker_only deploy yaml not found") @@ -2073,7 +2073,7 @@ class TestMingFlashOmniPipeline: def test_image_yaml_loads_and_merges(self): """deploy/ming_flash_omni_image.yaml parses and routes to the image pipeline.""" - deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "ming_flash_omni_image.yaml" + deploy_path = Path(get_deploy_config_path("ming_flash_omni_image.yaml")) if not deploy_path.exists(): pytest.skip("ming_flash_omni_image deploy yaml not found") @@ -2135,7 +2135,7 @@ class TestBaseConfigInheritance: def test_pure_inheritance_overlay(self, tmp_path): """An overlay with only ``base_config`` inherits everything.""" - base = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_omni_moe.yaml" + base = Path(get_deploy_config_path("qwen3_omni_moe.yaml")) if not base.exists(): pytest.skip("Base deploy config not found") @@ -2148,7 +2148,7 @@ class TestBaseConfigInheritance: def test_single_field_overlay(self, tmp_path): """An overlay overriding one stage field merges with the base.""" - base = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_omni_moe.yaml" + base = Path(get_deploy_config_path("qwen3_omni_moe.yaml")) if not base.exists(): pytest.skip("Base deploy config not found") @@ -2165,7 +2165,7 @@ class TestPlatformOverrides: """Test platform-specific deploy config overrides.""" def test_npu_overrides(self): - deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_omni_moe.yaml" + deploy_path = Path(get_deploy_config_path("qwen3_omni_moe.yaml")) if not deploy_path.exists(): pytest.skip("Deploy config not found") @@ -2179,7 +2179,7 @@ class TestPlatformOverrides: assert deploy.stages[2].enforce_eager is False def test_xpu_overrides(self): - deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_omni_moe.yaml" + deploy_path = Path(get_deploy_config_path("qwen3_omni_moe.yaml")) if not deploy_path.exists(): pytest.skip("Deploy config not found") @@ -2191,7 +2191,7 @@ class TestPlatformOverrides: assert deploy.stages[0].engine_extras.get("max_cudagraph_capture_size") == 0 def test_unknown_platform_noop(self): - deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_omni_moe.yaml" + deploy_path = Path(get_deploy_config_path("qwen3_omni_moe.yaml")) if not deploy_path.exists(): pytest.skip("Deploy config not found") @@ -2236,7 +2236,7 @@ class TestCLIOverrideFlow: """Test --stage-overrides JSON merge into StageConfig.""" def test_stage_overrides_merge(self): - deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_omni_moe.yaml" + deploy_path = Path(get_deploy_config_path("qwen3_omni_moe.yaml")) if not deploy_path.exists(): pytest.skip("Deploy config not found") @@ -2254,7 +2254,7 @@ class TestCLIOverrideFlow: assert stages[0].runtime_overrides["gpu_memory_utilization"] == 0.5 def test_global_override_applies_to_all(self): - deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_omni_moe.yaml" + deploy_path = Path(get_deploy_config_path("qwen3_omni_moe.yaml")) if not deploy_path.exists(): pytest.skip("Deploy config not found") @@ -2274,7 +2274,7 @@ class TestCLIOverrideFlow: class TestAuraOmniDeploy: def test_aura_omni_deploy_forces_pipeline_override(self): - deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "aura_omni.yaml" + deploy_path = Path(get_deploy_config_path("aura_omni.yaml")) deploy = load_deploy_config(deploy_path) assert deploy.pipeline == "aura_omni" @@ -2285,7 +2285,7 @@ class TestAuraOmniDeploy: stages, _ = StageConfigFactory._create_legacy_from_registry( pipeline_cfg, cli_overrides={}, - deploy_config_path=str(Path(__file__).parent.parent / "vllm_omni" / "deploy" / "aura_omni.yaml"), + deploy_config_path=get_deploy_config_path("aura_omni.yaml"), ) assert [stage.model_stage for stage in stages] == [ @@ -2599,7 +2599,12 @@ class TestSentinelDefaultPrecedence: from vllm_omni.config.stage_config import DeployConfig, merge_pipeline_deploy pipeline_path = ( - Path(__file__).parent.parent / "vllm_omni" / "model_executor" / "models" / "qwen3_omni" / "pipeline.py" + Path(__file__).parent.parent.parent + / "vllm_omni" + / "model_executor" + / "models" + / "qwen3_omni" + / "pipeline.py" ) pipeline = runpy.run_path(str(pipeline_path))["QWEN3_OMNI_PIPELINE"] @@ -2669,7 +2674,7 @@ class TestSamplingConstraintsPrecedence: """Test that pipeline sampling_constraints override deploy defaults.""" def test_constraints_win(self): - deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_omni_moe.yaml" + deploy_path = Path(get_deploy_config_path("qwen3_omni_moe.yaml")) if not deploy_path.exists(): pytest.skip("Deploy config not found") diff --git a/tests/conftest.py b/tests/conftest.py index 11619b782..b453bbcbf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,6 +3,7 @@ Root pytest entrypoint for the vLLM-Omni test suite. - `tests/conftest.py` stays thin: plugin registration + compatibility re-exports. - Importable utilities live under `tests/helpers/`. +- Helper unit tests live under `tests/helpers/tests/`. - Fixtures live under `tests/helpers/fixtures/` and are loaded via `pytest_plugins`. """ diff --git a/tests/core/sched/test_omni_scheduler_mixin_timeouts.py b/tests/core/sched/test_omni_scheduler_mixin_timeouts.py index 053c926cc..5fad4340c 100644 --- a/tests/core/sched/test_omni_scheduler_mixin_timeouts.py +++ b/tests/core/sched/test_omni_scheduler_mixin_timeouts.py @@ -18,6 +18,8 @@ import pytest from vllm_omni.core.sched.omni_scheduler_mixin import OmniSchedulerMixin +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + class _FakeCoordinator: def __init__(self, timed_out_ids): diff --git a/tests/core/test_prefix_cache.py b/tests/core/test_prefix_cache.py index 2111556f2..db7610496 100644 --- a/tests/core/test_prefix_cache.py +++ b/tests/core/test_prefix_cache.py @@ -3,6 +3,9 @@ import torch from vllm_omni.core.prefix_cache import OmniTensorPrefixCache +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + DEFAULT_SEQ_LEN = 15 NUM_BLOCKS = 10 BLOCK_SIZE = 4 diff --git a/tests/ar_diffusion/__init__.py b/tests/diffusion/ar_diffusion/__init__.py similarity index 100% rename from tests/ar_diffusion/__init__.py rename to tests/diffusion/ar_diffusion/__init__.py diff --git a/tests/ar_diffusion/test_capability_runner.py b/tests/diffusion/ar_diffusion/test_capability_runner.py similarity index 100% rename from tests/ar_diffusion/test_capability_runner.py rename to tests/diffusion/ar_diffusion/test_capability_runner.py diff --git a/tests/ar_diffusion/test_engine_routing.py b/tests/diffusion/ar_diffusion/test_engine_routing.py similarity index 98% rename from tests/ar_diffusion/test_engine_routing.py rename to tests/diffusion/ar_diffusion/test_engine_routing.py index 5972ffff4..19c4cb529 100644 --- a/tests/ar_diffusion/test_engine_routing.py +++ b/tests/diffusion/ar_diffusion/test_engine_routing.py @@ -21,6 +21,8 @@ from vllm_omni.experimental.ar_diffusion.engine import ( ARDiffusionEngine, ) +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + def _cfg(backend): return SimpleNamespace(engine_backend=backend) diff --git a/tests/ar_diffusion/test_kv_cache.py b/tests/diffusion/ar_diffusion/test_kv_cache.py similarity index 99% rename from tests/ar_diffusion/test_kv_cache.py rename to tests/diffusion/ar_diffusion/test_kv_cache.py index 9ca0e547d..d09c1c1b6 100644 --- a/tests/ar_diffusion/test_kv_cache.py +++ b/tests/diffusion/ar_diffusion/test_kv_cache.py @@ -25,6 +25,9 @@ from vllm_omni.experimental.ar_diffusion.kv_cache import ( ) from vllm_omni.experimental.ar_diffusion.kv_cache.paged import chunk_window_skipped_tokens +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + + BLOCK = 16 diff --git a/tests/ar_diffusion/test_kv_orchestration.py b/tests/diffusion/ar_diffusion/test_kv_orchestration.py similarity index 98% rename from tests/ar_diffusion/test_kv_orchestration.py rename to tests/diffusion/ar_diffusion/test_kv_orchestration.py index b7c77390c..32603bf88 100644 --- a/tests/ar_diffusion/test_kv_orchestration.py +++ b/tests/diffusion/ar_diffusion/test_kv_orchestration.py @@ -7,6 +7,7 @@ import torch from vllm_omni.experimental.ar_diffusion.capability import ARDiffusionKVBranchSpec from vllm_omni.experimental.ar_diffusion.kv_cache import ARDiffusionKVCache, ARDiffusionKVConfig +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] DIMS = dict( num_layers=2, num_kv_heads=4, diff --git a/tests/ar_diffusion/test_paged_attention.py b/tests/diffusion/ar_diffusion/test_paged_attention.py similarity index 94% rename from tests/ar_diffusion/test_paged_attention.py rename to tests/diffusion/ar_diffusion/test_paged_attention.py index b5c5e5870..0b66c01a0 100644 --- a/tests/ar_diffusion/test_paged_attention.py +++ b/tests/diffusion/ar_diffusion/test_paged_attention.py @@ -22,6 +22,9 @@ from vllm_omni.experimental.ar_diffusion.kv_cache import ( ) from vllm_omni.experimental.ar_diffusion.kv_cache.state import ARDiffusionKVState +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + + BLOCK = 16 N_HEADS = 4 HEAD_DIM = 64 @@ -385,17 +388,22 @@ def test_custom_op_compiles_fullgraph_without_recompile_on_value_change(): return out torch._dynamo.reset() - from torch._dynamo.testing import CompileCounter + try: + from torch._dynamo.testing import CompileCounter - counter = CompileCounter() + counter = CompileCounter() - def fn(inputs, q, k, v): - return paged_write_attn(inputs, q, k, v, None, None, HEAD_DIM**-0.5) * 1.0 + def fn(inputs, q, k, v): + return paged_write_attn(inputs, q, k, v, None, None, HEAD_DIM**-0.5) * 1.0 - compiled = torch.compile(fn, backend=counter, fullgraph=True) + compiled = torch.compile(fn, backend=counter, fullgraph=True) - run_one_forward(commit=True) # history grows between calls -> - run_one_forward(commit=True) # block-table VALUES change, shapes don't - run_one_forward(commit=False) + run_one_forward(commit=True) # history grows between calls -> + run_one_forward(commit=True) # block-table VALUES change, shapes don't + run_one_forward(commit=False) - assert counter.frame_count == 1, f"recompiled: frame_count={counter.frame_count}" + assert counter.frame_count == 1, f"recompiled: frame_count={counter.frame_count}" + finally: + # Leave a clean dynamo state for later suites in the same pytest process + # (e.g. model_executor transformers models). + torch._dynamo.reset() diff --git a/tests/ar_diffusion/test_pipeline_bridge.py b/tests/diffusion/ar_diffusion/test_pipeline_bridge.py similarity index 99% rename from tests/ar_diffusion/test_pipeline_bridge.py rename to tests/diffusion/ar_diffusion/test_pipeline_bridge.py index 92d490b92..ec60290b9 100644 --- a/tests/ar_diffusion/test_pipeline_bridge.py +++ b/tests/diffusion/ar_diffusion/test_pipeline_bridge.py @@ -12,6 +12,9 @@ from vllm_omni.experimental.ar_diffusion.kv_cache import ( ) from vllm_omni.experimental.ar_diffusion.kv_cache.state import ARDiffusionKVState +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + + BLOCK = 16 N_HEADS = 4 HEAD_DIM = 64 diff --git a/tests/ar_diffusion/test_slot_mapping.py b/tests/diffusion/ar_diffusion/test_slot_mapping.py similarity index 97% rename from tests/ar_diffusion/test_slot_mapping.py rename to tests/diffusion/ar_diffusion/test_slot_mapping.py index 061df4d9c..4c1f82537 100644 --- a/tests/ar_diffusion/test_slot_mapping.py +++ b/tests/diffusion/ar_diffusion/test_slot_mapping.py @@ -13,6 +13,9 @@ from vllm_omni.experimental.ar_diffusion.kv_cache import ( resident_block_ids, ) +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + + BLOCK = 16 diff --git a/tests/diffusion/attention/test_attention_config.py b/tests/diffusion/attention/test_attention_config.py index 26a93e901..adf13d51c 100644 --- a/tests/diffusion/attention/test_attention_config.py +++ b/tests/diffusion/attention/test_attention_config.py @@ -288,6 +288,13 @@ class TestBuildAttentionConfig: class TestOmniDiffusionConfigAttentionParsing: """Test OmniDiffusionConfig attention shorthand and structured config.""" + @pytest.fixture(autouse=True) + def _clear_diffusion_attention_backend_env(self, monkeypatch): + # OmniDiffusionConfig.__post_init__ applies DIFFUSION_ATTENTION_BACKEND via + # build_attention_config(); clear it so these tests assert config defaults, + # not whatever the process inherited from CI / sibling tests. + monkeypatch.delenv("DIFFUSION_ATTENTION_BACKEND", raising=False) + def test_diffusion_attention_backend_sets_default(self): config = OmniDiffusionConfig.from_kwargs(diffusion_attention_backend="SAGE_ATTN") assert isinstance(config.diffusion_attention_config, AttentionConfig) diff --git a/tests/diffusion/attention/test_attention_sp.py b/tests/diffusion/attention/test_attention_sp.py index e32c4c5e4..0f1faacd1 100644 --- a/tests/diffusion/attention/test_attention_sp.py +++ b/tests/diffusion/attention/test_attention_sp.py @@ -49,6 +49,8 @@ from vllm_omni.diffusion.distributed.parallel_state import ( from vllm_omni.diffusion.forward_context import set_forward_context from vllm_omni.platforms import current_omni_platform +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + def update_environment_variables(envs_dict: dict[str, str]): """Update multiple environment variables with logging.""" diff --git a/tests/diffusion/attention/test_flash_attn.py b/tests/diffusion/attention/test_flash_attn.py index 5ffd6684a..fbe037274 100644 --- a/tests/diffusion/attention/test_flash_attn.py +++ b/tests/diffusion/attention/test_flash_attn.py @@ -18,6 +18,9 @@ from vllm_omni.diffusion.attention.backends.sdpa import SDPAImpl from vllm_omni.diffusion.attention.backends.utils import fa # noqa: E402 from vllm_omni.platforms import current_omni_platform +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + + is_gpu = current_omni_platform.is_cuda_alike() or current_omni_platform.is_xpu() HAS_FLASH_ATTN = fa.HAS_FLASH_ATTN flash_attn_func = fa.flash_attn_func # noqa: N813 diff --git a/tests/diffusion/attention/test_piecewise_attn.py b/tests/diffusion/attention/test_piecewise_attn.py index 00b98ac64..05967e28f 100644 --- a/tests/diffusion/attention/test_piecewise_attn.py +++ b/tests/diffusion/attention/test_piecewise_attn.py @@ -27,6 +27,9 @@ from vllm_omni.diffusion.attention.backends.utils.piecewise_attn import ( piecewise_attn, ) +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + + DEVICE = torch.device("cpu") diff --git a/tests/diffusion/attention/test_sage_attn3.py b/tests/diffusion/attention/test_sage_attn3.py index e4d913974..5c46ea5d0 100644 --- a/tests/diffusion/attention/test_sage_attn3.py +++ b/tests/diffusion/attention/test_sage_attn3.py @@ -9,6 +9,9 @@ import pytest import torch from vllm.platforms import current_platform +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + + SAGE_ATTN3_MODULE = "vllm_omni.diffusion.attention.backends.sage_attn3" diff --git a/tests/diffusion/attention/test_ulysses_uaa.py b/tests/diffusion/attention/test_ulysses_uaa.py index a9e2021c3..6bb588156 100644 --- a/tests/diffusion/attention/test_ulysses_uaa.py +++ b/tests/diffusion/attention/test_ulysses_uaa.py @@ -22,6 +22,8 @@ from vllm_omni.diffusion.distributed.parallel_state import ( from vllm_omni.diffusion.forward_context import get_forward_context, set_forward_context from vllm_omni.platforms import current_omni_platform +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + def _find_free_port() -> int: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: diff --git a/tests/diffusion/distributed/test_cfg_parallel.py b/tests/diffusion/distributed/test_cfg_parallel.py index af81f00ef..8991ddec3 100644 --- a/tests/diffusion/distributed/test_cfg_parallel.py +++ b/tests/diffusion/distributed/test_cfg_parallel.py @@ -1,18 +1,23 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for CFG (Classifier-Free Guidance) parallel functionality. +"""Tests for CFG (Classifier-Free Guidance) parallel functionality. -This test verifies that predict_noise_maybe_with_cfg and -predict_noise_with_multi_branch_cfg produce numerically equivalent results -with and without CFG parallel using fixed random inputs. +CPU tests mock the CFG process group and validate mixin logic without GPUs. +Nightly GPU parity tests compare sequential CFG (cfg_parallel_size=1) against +real multi-GPU CFG parallel under fixed seeds. """ +from __future__ import annotations + import os +from typing import Any import pytest import torch -from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin +from tests.helpers.mark import hardware_marks, hardware_test +from vllm_omni.diffusion.distributed import parallel_state +from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin, _dispatch_branches, _wrap from vllm_omni.diffusion.distributed.parallel_state import ( destroy_distributed_env, get_classifier_free_guidance_rank, @@ -22,445 +27,105 @@ from vllm_omni.diffusion.distributed.parallel_state import ( ) from vllm_omni.platforms import current_omni_platform +_L4_TWO_GPU = hardware_marks(res={"cuda": "L4"}, num_cards=2) +_L4_THREE_GPU = hardware_marks(res={"cuda": "L4"}, num_cards=3) -def update_environment_variables(envs_dict: dict[str, str]): - """Update multiple environment variables.""" - for k, v in envs_dict.items(): - os.environ[k] = v + +def _set_random_seeds(seed: int) -> None: + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + + +def _init_model_weights(module: torch.nn.Module, seed: int) -> None: + _set_random_seeds(seed) + for param in module.parameters(): + torch.nn.init.normal_(param, mean=0.0, std=0.02) + + +def _cfg_close_tolerance(dtype: torch.dtype) -> tuple[float, float]: + if dtype == torch.float32: + return 1e-5, 1e-5 + if dtype == torch.bfloat16: + return 1e-2, 1e-2 + return 1e-3, 1e-3 + + +def _assert_cfg_outputs_close( + actual: torch.Tensor, + expected: torch.Tensor, + *, + dtype: torch.dtype, + msg: str, +) -> None: + rtol, atol = _cfg_close_tolerance(dtype) + torch.testing.assert_close(actual, expected, rtol=rtol, atol=atol, msg=msg) + + +def _update_environment_variables(envs_dict: dict[str, str]) -> None: + for key, value in envs_dict.items(): + os.environ[key] = value class SimpleTransformer(torch.nn.Module): - """Simple transformer model for testing with random initialization. - - Contains: - - Input projection (conv to hidden_dim) - - QKV projection layers - - Self-attention layer - - Output projection - """ - def __init__(self, in_channels: int = 4, hidden_dim: int = 128, num_heads: int = 8): super().__init__() self.hidden_dim = hidden_dim self.num_heads = num_heads self.head_dim = hidden_dim // num_heads + assert hidden_dim % num_heads == 0 - assert hidden_dim % num_heads == 0, "hidden_dim must be divisible by num_heads" - - # Input projection: (B, C, H, W) -> (B, hidden_dim, H, W) self.input_proj = torch.nn.Conv2d(in_channels, hidden_dim, 1) - - # QKV projection layers self.q_proj = torch.nn.Linear(hidden_dim, hidden_dim) self.k_proj = torch.nn.Linear(hidden_dim, hidden_dim) self.v_proj = torch.nn.Linear(hidden_dim, hidden_dim) - - # Output projection after attention self.out_proj = torch.nn.Linear(hidden_dim, hidden_dim) - - # Final output projection: (B, hidden_dim, H, W) -> (B, C, H, W) self.final_proj = torch.nn.Conv2d(hidden_dim, in_channels, 1) - - # Layer norm self.norm1 = torch.nn.LayerNorm(hidden_dim) self.norm2 = torch.nn.LayerNorm(hidden_dim) def forward(self, x: torch.Tensor, **kwargs) -> tuple[torch.Tensor]: - """Forward pass with self-attention. - - Args: - x: Input tensor of shape (B, C, H, W) - - Returns: - Output tensor of shape (B, C, H, W) - """ - B, C, H, W = x.shape - - # Input projection - x = self.input_proj(x) # (B, hidden_dim, H, W) - - # Reshape to sequence: (B, hidden_dim, H, W) -> (B, H*W, hidden_dim) - x = x.flatten(2).transpose(1, 2) # (B, H*W, hidden_dim) - - # Self-attention with residual connection + del kwargs + batch_size, _channels, height, width = x.shape + x = self.input_proj(x) + x = x.flatten(2).transpose(1, 2) residual = x x = self.norm1(x) - - # QKV projection - q = self.q_proj(x) # (B, H*W, hidden_dim) - k = self.k_proj(x) # (B, H*W, hidden_dim) - v = self.v_proj(x) # (B, H*W, hidden_dim) - - # Reshape for multi-head attention: (B, H*W, hidden_dim) -> (B, num_heads, H*W, head_dim) - seq_len = H * W - q = q.view(B, seq_len, self.num_heads, self.head_dim).transpose(1, 2) - k = k.view(B, seq_len, self.num_heads, self.head_dim).transpose(1, 2) - v = v.view(B, seq_len, self.num_heads, self.head_dim).transpose(1, 2) - - # Scaled dot-product attention + q = self.q_proj(x) + k = self.k_proj(x) + v = self.v_proj(x) + seq_len = height * width + q = q.view(batch_size, seq_len, self.num_heads, self.head_dim).transpose(1, 2) + k = k.view(batch_size, seq_len, self.num_heads, self.head_dim).transpose(1, 2) + v = v.view(batch_size, seq_len, self.num_heads, self.head_dim).transpose(1, 2) scale = self.head_dim**-0.5 - attn_scores = torch.matmul(q, k.transpose(-2, -1)) * scale # (B, num_heads, H*W, H*W) + attn_scores = torch.matmul(q, k.transpose(-2, -1)) * scale attn_weights = torch.nn.functional.softmax(attn_scores, dim=-1) attn_output = torch.matmul(attn_weights, v) - attn_output = attn_output.transpose(1, 2).contiguous().view(B, seq_len, self.hidden_dim) - + attn_output = attn_output.transpose(1, 2).contiguous().view(batch_size, seq_len, self.hidden_dim) attn_output = self.out_proj(attn_output) - x = residual + attn_output residual = x x = self.norm2(x) x = residual + x - x = x.transpose(1, 2).view(B, self.hidden_dim, H, W) - - out = self.final_proj(x) - - return (out,) + x = x.transpose(1, 2).view(batch_size, self.hidden_dim, height, width) + return (self.final_proj(x),) class TestCFGPipeline(CFGParallelMixin): - """Test pipeline using CFGParallelMixin.""" - def __init__(self, in_channels: int = 4, hidden_dim: int = 128, seed: int = 42): - # Set seed BEFORE creating transformer to ensure consistent layer initialization - torch.manual_seed(seed) - if torch.cuda.is_available(): - torch.cuda.manual_seed_all(seed) - + _set_random_seeds(seed) self.transformer = SimpleTransformer(in_channels, hidden_dim) - - # Re-initialize all parameters with fixed seed for full reproducibility - torch.manual_seed(seed) - if torch.cuda.is_available(): - torch.cuda.manual_seed_all(seed) - for param in self.transformer.parameters(): - torch.nn.init.normal_(param, mean=0.0, std=0.02) - - -def _test_cfg_parallel_worker( - local_rank: int, - world_size: int, - cfg_parallel_size: int, - dtype: torch.dtype, - test_config: dict, - result_queue: torch.multiprocessing.Queue, -): - """Worker function for CFG parallel test.""" - device = torch.device(f"{current_omni_platform.device_type}:{local_rank}") - current_omni_platform.set_device(device) - - update_environment_variables( - { - "RANK": str(local_rank), - "LOCAL_RANK": str(local_rank), - "WORLD_SIZE": str(world_size), - "MASTER_ADDR": "localhost", - "MASTER_PORT": "29502", - } - ) - - init_distributed_environment() - initialize_model_parallel(cfg_parallel_size=cfg_parallel_size) - - cfg_rank = get_classifier_free_guidance_rank() - cfg_world_size = get_classifier_free_guidance_world_size() - - assert cfg_world_size == cfg_parallel_size - - # Create pipeline with same seed to ensure identical model weights across all ranks - # Note: model_seed is set inside TestCFGPipeline.__init__ - pipeline = TestCFGPipeline( - in_channels=test_config["channels"], - hidden_dim=test_config["hidden_dim"], - seed=test_config["model_seed"], - ) - pipeline.transformer = pipeline.transformer.to(device=device, dtype=dtype) - pipeline.transformer.eval() # Set to eval mode for deterministic behavior - - # Create fixed inputs with explicit seed setting for reproducibility - # Set both CPU and CUDA seeds to ensure identical inputs across all ranks - torch.manual_seed(test_config["input_seed"]) - if torch.cuda.is_available(): - torch.cuda.manual_seed_all(test_config["input_seed"]) - - batch_size = test_config["batch_size"] - channels = test_config["channels"] - height = test_config["height"] - width = test_config["width"] - - # Positive input - positive_input = torch.randn(batch_size, channels, height, width, dtype=dtype, device=device) - - # Negative input with different seed - torch.manual_seed(test_config["input_seed"] + 1) - if torch.cuda.is_available(): - torch.cuda.manual_seed_all(test_config["input_seed"] + 1) - negative_input = torch.randn(batch_size, channels, height, width, dtype=dtype, device=device) - - # Prepare kwargs for predict_noise_maybe_with_cfg - positive_kwargs = {"x": positive_input} - negative_kwargs = {"x": negative_input} - - with torch.no_grad(): - # Call predict_noise_maybe_with_cfg - noise_pred = pipeline.predict_noise_maybe_with_cfg( - do_true_cfg=True, - true_cfg_scale=test_config["cfg_scale"], - positive_kwargs=positive_kwargs, - negative_kwargs=negative_kwargs, - cfg_normalize=test_config["cfg_normalize"], - kwargs=test_config["kwargs"], - ) - - # CFG parallel returns the combined prediction on every rank. - assert noise_pred is not None - result_queue.put((cfg_rank, noise_pred.cpu())) - - destroy_distributed_env() - - -def _test_cfg_sequential_worker( - local_rank: int, - world_size: int, - dtype: torch.dtype, - test_config: dict, - result_queue: torch.multiprocessing.Queue, -): - """Worker function for sequential CFG test (baseline).""" - device = torch.device(f"{current_omni_platform.device_type}:{local_rank}") - current_omni_platform.set_device(device) - - update_environment_variables( - { - "RANK": str(local_rank), - "LOCAL_RANK": str(local_rank), - "WORLD_SIZE": str(world_size), - "MASTER_ADDR": "localhost", - "MASTER_PORT": "29503", - } - ) - - init_distributed_environment() - initialize_model_parallel(cfg_parallel_size=1) # No CFG parallel - - cfg_world_size = get_classifier_free_guidance_world_size() - assert cfg_world_size == 1 - - # Create pipeline with same seed to ensure identical model weights as CFG parallel - # Note: model_seed is set inside TestCFGPipeline.__init__ - pipeline = TestCFGPipeline( - in_channels=test_config["channels"], - hidden_dim=test_config["hidden_dim"], - seed=test_config["model_seed"], - ) - pipeline.transformer = pipeline.transformer.to(device=device, dtype=dtype) - pipeline.transformer.eval() - - # Create fixed inputs (same seed as CFG parallel to ensure identical inputs) - # Set both CPU and CUDA seeds for full reproducibility - torch.manual_seed(test_config["input_seed"]) - if torch.cuda.is_available(): - torch.cuda.manual_seed_all(test_config["input_seed"]) - - batch_size = test_config["batch_size"] - channels = test_config["channels"] - height = test_config["height"] - width = test_config["width"] - - # Positive input - positive_input = torch.randn(batch_size, channels, height, width, dtype=dtype, device=device) - - # Negative input with different seed - torch.manual_seed(test_config["input_seed"] + 1) - if torch.cuda.is_available(): - torch.cuda.manual_seed_all(test_config["input_seed"] + 1) - negative_input = torch.randn(batch_size, channels, height, width, dtype=dtype, device=device) - - positive_kwargs = {"x": positive_input} - negative_kwargs = {"x": negative_input} - - with torch.no_grad(): - noise_pred = pipeline.predict_noise_maybe_with_cfg( - do_true_cfg=True, - true_cfg_scale=test_config["cfg_scale"], - positive_kwargs=positive_kwargs, - negative_kwargs=negative_kwargs, - cfg_normalize=test_config["cfg_normalize"], - kwargs=test_config["kwargs"], - ) - - # Sequential CFG always returns output - assert noise_pred is not None - result_queue.put(noise_pred.cpu()) - - destroy_distributed_env() - - -@pytest.mark.parametrize("cfg_parallel_size", [2]) -@pytest.mark.parametrize("dtype", [torch.bfloat16]) -@pytest.mark.parametrize("batch_size", [2]) -@pytest.mark.parametrize("cfg_normalize", [False, True]) -@pytest.mark.parametrize("kwargs", [None, {"step_i": 1}]) -def test_predict_noise_maybe_with_cfg( - cfg_parallel_size: int, dtype: torch.dtype, batch_size: int, cfg_normalize: bool, kwargs: dict | None -): - """ - Test that predict_noise_maybe_with_cfg produces identical results - with and without CFG parallel. - - Args: - cfg_parallel_size: Number of GPUs for CFG parallel - dtype: Data type for computation - batch_size: Batch size for testing - cfg_normalize: Whether to normalize CFG output - """ - available_gpus = current_omni_platform.get_device_count() - if available_gpus < cfg_parallel_size: - pytest.skip(f"Test requires {cfg_parallel_size} GPUs but only {available_gpus} available") - - test_config = { - "batch_size": batch_size, - "channels": 4, - "height": 16, - "width": 16, - "hidden_dim": 128, - "cfg_scale": 7.5, - "cfg_normalize": cfg_normalize, - "model_seed": 42, # Fixed seed for model initialization - "input_seed": 123, # Fixed seed for input generation - "kwargs": kwargs, # Additional kwargs to test passing through CFG parallel - } - - mp_context = torch.multiprocessing.get_context("spawn") - - manager = mp_context.Manager() - baseline_queue = manager.Queue() - cfg_parallel_queue = manager.Queue() - - # Run baseline (sequential CFG) on single GPU - torch.multiprocessing.spawn( - _test_cfg_sequential_worker, - args=(1, dtype, test_config, baseline_queue), - nprocs=1, - ) - - # Run CFG parallel on multiple GPUs - torch.multiprocessing.spawn( - _test_cfg_parallel_worker, - args=(cfg_parallel_size, cfg_parallel_size, dtype, test_config, cfg_parallel_queue), - nprocs=cfg_parallel_size, - ) - - # Get results from queues - baseline_output = baseline_queue.get() - cfg_parallel_outputs = [cfg_parallel_queue.get() for _ in range(cfg_parallel_size)] - cfg_parallel_outputs.sort(key=lambda item: item[0]) - cfg_parallel_output = cfg_parallel_outputs[0][1] - - for cfg_rank, rank_output in cfg_parallel_outputs[1:]: - torch.testing.assert_close( - rank_output, - cfg_parallel_output, - rtol=0, - atol=0, - msg=f"CFG parallel ranks produced different outputs (rank 0 vs rank {cfg_rank})", - ) - - # Verify shapes match - assert baseline_output.shape == cfg_parallel_output.shape, ( - f"Shape mismatch: baseline {baseline_output.shape} vs CFG parallel {cfg_parallel_output.shape}" - ) - - # Verify numerical equivalence with appropriate tolerances - if dtype == torch.float32: - rtol, atol = 1e-5, 1e-5 - elif dtype == torch.bfloat16: - rtol, atol = 1e-2, 1e-2 - else: - rtol, atol = 1e-3, 1e-3 - - torch.testing.assert_close( - cfg_parallel_output, - baseline_output, - rtol=rtol, - atol=atol, - msg=( - f"CFG parallel output differs from sequential CFG\n" - f" dtype={dtype}, batch_size={batch_size}, cfg_normalize={cfg_normalize}\n" - f" Max diff: {(cfg_parallel_output - baseline_output).abs().max().item():.6e}" - ), - ) - - print( - f"✓ Test passed: cfg_size={cfg_parallel_size}, dtype={dtype}, " - f"batch_size={batch_size}, cfg_normalize={cfg_normalize}" - ) - - -@pytest.mark.parametrize("dtype", [torch.bfloat16]) -def test_predict_noise_without_cfg(dtype: torch.dtype): - """ - Test predict_noise_maybe_with_cfg when do_true_cfg=False. - - When CFG is disabled, only the positive branch should be computed. - This test runs on a single GPU without distributed environment. - """ - available_gpus = current_omni_platform.get_device_count() - if available_gpus < 1: - pytest.skip("Test requires at least 1 GPU") - - device = torch.device(f"{current_omni_platform.device_type}:0") - current_omni_platform.set_device(device) - - # Create pipeline without distributed environment - pipeline = TestCFGPipeline(in_channels=4, hidden_dim=128, seed=42) - pipeline.transformer = pipeline.transformer.to(device=device, dtype=dtype) - pipeline.transformer.eval() - - # Set seed for input generation - torch.manual_seed(123) - if torch.cuda.is_available(): - torch.cuda.manual_seed_all(123) - positive_input = torch.randn(1, 4, 16, 16, dtype=dtype, device=device) - - with torch.no_grad(): - noise_pred = pipeline.predict_noise_maybe_with_cfg( - do_true_cfg=False, # No CFG - true_cfg_scale=7.5, - positive_kwargs={"x": positive_input}, - negative_kwargs=None, - cfg_normalize=False, - ) - - # Should always return output when do_true_cfg=False - assert noise_pred is not None - assert noise_pred.shape == (1, 4, 16, 16) - - print(f"✓ Test passed: predict_noise without CFG (dtype={dtype})") + _init_model_weights(self.transformer, seed) class MultiBranchTestPipeline(CFGParallelMixin): - """Test pipeline with custom 3-branch combine logic (like OmniGen2).""" - def __init__(self, in_channels: int = 4, hidden_dim: int = 128, seed: int = 42): - torch.manual_seed(seed) - if torch.cuda.is_available(): - torch.cuda.manual_seed_all(seed) - + _set_random_seeds(seed) self.transformer = SimpleTransformer(in_channels, hidden_dim) - - torch.manual_seed(seed) - if torch.cuda.is_available(): - torch.cuda.manual_seed_all(seed) - for param in self.transformer.parameters(): - torch.nn.init.normal_(param, mean=0.0, std=0.02) + _init_model_weights(self.transformer, seed) def combine_multi_branch_cfg_noise(self, predictions, true_cfg_scale, cfg_normalize=False): - """N-branch combine with weighted sum for testing. - - - 2-branch: standard CFG formula (true_cfg_scale is float) - - 3-branch: OmniGen2-style dual guidance scale (true_cfg_scale is dict) - - 4-branch: DreamID-style weighted sum (true_cfg_scale is dict) - """ if len(predictions) == 4: text_scale = true_cfg_scale["text"] image_scale = true_cfg_scale["image"] @@ -486,6 +151,561 @@ class MultiBranchTestPipeline(CFGParallelMixin): return combined +class FakeCfgGroup: + def __init__(self, *, world_size: int, rank_in_group: int, rank_tensors: list[torch.Tensor]): + self.world_size = world_size + self.rank_in_group = rank_in_group + self._rank_tensors = rank_tensors + + def all_gather( + self, + input_: torch.Tensor, + dim: int = 0, + separate_tensors: bool = False, + ) -> torch.Tensor | list[torch.Tensor]: + del dim, input_ + assert separate_tensors + return [tensor.clone() for tensor in self._rank_tensors] + + +class FakeMultiBranchCfgGroup: + def __init__( + self, + *, + world_size: int, + rank_in_group: int, + slots_per_rank: list[list[tuple[torch.Tensor, ...]]], + ): + self.world_size = world_size + self.rank_in_group = rank_in_group + self._slots_per_rank = slots_per_rank + # Production path re-runs predict_noise, so gathered tensors are value-equal + # but not identity-equal to the precomputed slot table. Index by call order + # instead: slot-major then element-major, matching _predict_multi_branch_parallel. + self._call_idx = 0 + + def all_gather( + self, + input_: torch.Tensor, + dim: int = 0, + separate_tensors: bool = False, + ) -> torch.Tensor | list[torch.Tensor]: + del dim, input_ + assert separate_tensors + n_elems = len(self._slots_per_rank[0][0]) + slot_idx, elem_idx = divmod(self._call_idx, n_elems) + self._call_idx += 1 + assert slot_idx < len(self._slots_per_rank[self.rank_in_group]), ( + f"all_gather call {self._call_idx} exceeds slot table " + f"(slot_idx={slot_idx}, n_slots={len(self._slots_per_rank[self.rank_in_group])})" + ) + return [self._slots_per_rank[rank][slot_idx][elem_idx].clone() for rank in range(self.world_size)] + + +def _patch_cfg_state( + monkeypatch: pytest.MonkeyPatch, + *, + world_size: int, + rank: int, + fake_group: FakeCfgGroup | FakeMultiBranchCfgGroup, +) -> None: + # Patch both the source module and cfg_parallel's bound imports. + # cfg_parallel does `from ...parallel_state import get_cfg_group`, so + # patching parallel_state alone does not affect CFGParallelMixin. + from vllm_omni.diffusion.distributed import cfg_parallel as cfg_parallel_mod + + for module in (parallel_state, cfg_parallel_mod): + monkeypatch.setattr(module, "get_classifier_free_guidance_world_size", lambda: world_size) + monkeypatch.setattr(module, "get_classifier_free_guidance_rank", lambda: rank) + monkeypatch.setattr(module, "get_cfg_group", lambda: fake_group) + + +def _make_two_branch_inputs( + *, + batch_size: int, + channels: int, + height: int, + width: int, + dtype: torch.dtype, + device: torch.device, + input_seed: int, +) -> tuple[dict[str, Any], dict[str, Any]]: + _set_random_seeds(input_seed) + positive = torch.randn(batch_size, channels, height, width, dtype=dtype, device=device) + _set_random_seeds(input_seed + 1) + negative = torch.randn(batch_size, channels, height, width, dtype=dtype, device=device) + return {"x": positive}, {"x": negative} + + +def _make_multi_branch_inputs( + *, + n_branches: int, + batch_size: int, + channels: int, + height: int, + width: int, + dtype: torch.dtype, + device: torch.device, + input_seed: int, +) -> list[dict[str, Any]]: + branches_kwargs: list[dict[str, Any]] = [] + for branch_idx in range(n_branches): + _set_random_seeds(input_seed + branch_idx) + x = torch.randn(batch_size, channels, height, width, dtype=dtype, device=device) + branches_kwargs.append({"x": x}) + return branches_kwargs + + +def _multi_branch_cfg_scale(n_branches: int) -> float | dict[str, float]: + if n_branches == 2: + return 5.0 + if n_branches == 3: + return {"text": 5.0, "image": 2.0} + return {"text": 5.0, "image": 2.0, "vid_ref": 1.5} + + +def _build_multi_branch_slot_table( + *, + n_branches: int, + cfg_world_size: int, + branch_preds: list[torch.Tensor | tuple[torch.Tensor, ...]], +) -> list[list[tuple[torch.Tensor, ...]]]: + assignments = _dispatch_branches(n_branches, cfg_world_size) + max_per_rank = max(len(branch_ids) for branch_ids in assignments) + ref_pred = _wrap(branch_preds[0]) + slots_per_rank: list[list[tuple[torch.Tensor, ...]]] = [] + for branch_ids in assignments: + slots = [_wrap(branch_preds[branch_id]) for branch_id in branch_ids] + while len(slots) < max_per_rank: + slots.append(tuple(torch.zeros_like(tensor) for tensor in ref_pred)) + slots_per_rank.append(slots) + return slots_per_rank + + +def _run_two_branch_cfg_parallel_mock( + pipeline: TestCFGPipeline, + monkeypatch: pytest.MonkeyPatch, + *, + cfg_world_size: int, + positive_kwargs: dict[str, Any], + negative_kwargs: dict[str, Any], + true_cfg_scale: float, + cfg_normalize: bool, + extra_kwargs: dict[str, Any] | None, +) -> torch.Tensor: + with torch.no_grad(): + pos_tensor = _wrap(pipeline.predict_noise(**positive_kwargs))[0] + neg_tensor = _wrap(pipeline.predict_noise(**negative_kwargs))[0] + + outputs: list[torch.Tensor] = [] + for cfg_rank in range(cfg_world_size): + fake_group = FakeCfgGroup( + world_size=cfg_world_size, + rank_in_group=cfg_rank, + rank_tensors=[pos_tensor, neg_tensor], + ) + _patch_cfg_state(monkeypatch, world_size=cfg_world_size, rank=cfg_rank, fake_group=fake_group) + with torch.no_grad(): + output = pipeline.predict_noise_maybe_with_cfg( + do_true_cfg=True, + true_cfg_scale=true_cfg_scale, + positive_kwargs=positive_kwargs, + negative_kwargs=negative_kwargs, + cfg_normalize=cfg_normalize, + kwargs=extra_kwargs, + ) + assert output is not None + outputs.append(output) + + for rank_output in outputs[1:]: + torch.testing.assert_close( + rank_output, + outputs[0], + rtol=0, + atol=0, + msg="CFG parallel ranks produced different outputs in mock path", + ) + return outputs[0] + + +def _run_multi_branch_cfg_parallel_mock( + pipeline: MultiBranchTestPipeline, + monkeypatch: pytest.MonkeyPatch, + *, + cfg_world_size: int, + branches_kwargs: list[dict[str, Any]], + true_cfg_scale: float | dict[str, float], + cfg_normalize: bool, +) -> torch.Tensor: + with torch.no_grad(): + branch_preds = [pipeline.predict_noise(**kwargs) for kwargs in branches_kwargs] + slots_per_rank = _build_multi_branch_slot_table( + n_branches=len(branches_kwargs), + cfg_world_size=cfg_world_size, + branch_preds=branch_preds, + ) + + outputs: list[torch.Tensor] = [] + for cfg_rank in range(cfg_world_size): + fake_group = FakeMultiBranchCfgGroup( + world_size=cfg_world_size, + rank_in_group=cfg_rank, + slots_per_rank=slots_per_rank, + ) + _patch_cfg_state(monkeypatch, world_size=cfg_world_size, rank=cfg_rank, fake_group=fake_group) + with torch.no_grad(): + output = pipeline.predict_noise_with_multi_branch_cfg( + do_true_cfg=True, + true_cfg_scale=true_cfg_scale, + branches_kwargs=branches_kwargs, + cfg_normalize=cfg_normalize, + ) + assert output is not None + outputs.append(output) + + for rank_output in outputs[1:]: + torch.testing.assert_close( + rank_output, + outputs[0], + rtol=0, + atol=0, + msg="Multi-branch CFG parallel ranks produced different outputs in mock path", + ) + return outputs[0] + + +def _build_pipeline_on_device( + pipeline_cls: type, + *, + hidden_dim: int, + model_seed: int, + dtype: torch.dtype, + device: torch.device, +): + pipeline = pipeline_cls(in_channels=4, hidden_dim=hidden_dim, seed=model_seed) + pipeline.transformer = pipeline.transformer.to(device=device, dtype=dtype) + pipeline.transformer.eval() + return pipeline + + +# --------------------------------------------------------------------------- +# CPU: mixin logic via mocked CFG groups (no GPU / no torch.distributed spawn) +# --------------------------------------------------------------------------- + + +@pytest.mark.core_model +@pytest.mark.diffusion +@pytest.mark.cpu +@pytest.mark.parametrize("batch_size", [2]) +@pytest.mark.parametrize("cfg_normalize", [False, True]) +@pytest.mark.parametrize("extra_kwargs", [None, {"step_i": 1}]) +def test_predict_noise_with_cfg( + monkeypatch: pytest.MonkeyPatch, + batch_size: int, + cfg_normalize: bool, + extra_kwargs: dict[str, Any] | None, +): + dtype = torch.float32 + device = torch.device("cpu") + pipeline = _build_pipeline_on_device(TestCFGPipeline, hidden_dim=128, model_seed=42, dtype=dtype, device=device) + positive_kwargs, negative_kwargs = _make_two_branch_inputs( + batch_size=batch_size, + channels=4, + height=16, + width=16, + dtype=dtype, + device=device, + input_seed=123, + ) + + _patch_cfg_state( + monkeypatch, + world_size=1, + rank=0, + fake_group=FakeCfgGroup(world_size=1, rank_in_group=0, rank_tensors=[]), + ) + with torch.no_grad(): + baseline = pipeline.predict_noise_maybe_with_cfg( + do_true_cfg=True, + true_cfg_scale=7.5, + positive_kwargs=positive_kwargs, + negative_kwargs=negative_kwargs, + cfg_normalize=cfg_normalize, + kwargs=extra_kwargs, + ) + + parallel = _run_two_branch_cfg_parallel_mock( + pipeline, + monkeypatch, + cfg_world_size=2, + positive_kwargs=positive_kwargs, + negative_kwargs=negative_kwargs, + true_cfg_scale=7.5, + cfg_normalize=cfg_normalize, + extra_kwargs=extra_kwargs, + ) + assert baseline is not None + _assert_cfg_outputs_close( + parallel, + baseline, + dtype=dtype, + msg=f"mock CFG parallel differs from sequential (cfg_normalize={cfg_normalize})", + ) + + +@pytest.mark.core_model +@pytest.mark.diffusion +@pytest.mark.cpu +def test_predict_noise_without_cfg(): + dtype = torch.float32 + device = torch.device("cpu") + pipeline = _build_pipeline_on_device(TestCFGPipeline, hidden_dim=128, model_seed=42, dtype=dtype, device=device) + positive_kwargs, _ = _make_two_branch_inputs( + batch_size=1, + channels=4, + height=16, + width=16, + dtype=dtype, + device=device, + input_seed=123, + ) + + with torch.no_grad(): + noise_pred = pipeline.predict_noise_maybe_with_cfg( + do_true_cfg=False, + true_cfg_scale=7.5, + positive_kwargs=positive_kwargs, + negative_kwargs=None, + cfg_normalize=False, + ) + + assert noise_pred is not None + assert noise_pred.shape == (1, 4, 16, 16) + + +@pytest.mark.core_model +@pytest.mark.diffusion +@pytest.mark.cpu +@pytest.mark.parametrize( + "cfg_parallel_size,n_branches", + [(2, 2), (2, 3), (3, 3), (2, 4)], +) +@pytest.mark.parametrize("batch_size", [2]) +@pytest.mark.parametrize("cfg_normalize", [False, True]) +def test_predict_noise_with_multi_branch_cfg( + monkeypatch: pytest.MonkeyPatch, + cfg_parallel_size: int, + n_branches: int, + batch_size: int, + cfg_normalize: bool, +): + dtype = torch.float32 + device = torch.device("cpu") + pipeline = _build_pipeline_on_device( + MultiBranchTestPipeline, + hidden_dim=128, + model_seed=42, + dtype=dtype, + device=device, + ) + branches_kwargs = _make_multi_branch_inputs( + n_branches=n_branches, + batch_size=batch_size, + channels=4, + height=16, + width=16, + dtype=dtype, + device=device, + input_seed=123, + ) + cfg_scale = _multi_branch_cfg_scale(n_branches) + + _patch_cfg_state( + monkeypatch, + world_size=1, + rank=0, + fake_group=FakeCfgGroup(world_size=1, rank_in_group=0, rank_tensors=[]), + ) + with torch.no_grad(): + baseline = pipeline.predict_noise_with_multi_branch_cfg( + do_true_cfg=True, + true_cfg_scale=cfg_scale, + branches_kwargs=branches_kwargs, + cfg_normalize=cfg_normalize, + ) + + parallel = _run_multi_branch_cfg_parallel_mock( + pipeline, + monkeypatch, + cfg_world_size=cfg_parallel_size, + branches_kwargs=branches_kwargs, + true_cfg_scale=cfg_scale, + cfg_normalize=cfg_normalize, + ) + assert baseline is not None + _assert_cfg_outputs_close( + parallel, + baseline, + dtype=dtype, + msg=( + f"mock multi-branch CFG parallel differs from sequential " + f"(n_branches={n_branches}, cfg_parallel_size={cfg_parallel_size})" + ), + ) + + +@pytest.mark.core_model +@pytest.mark.diffusion +@pytest.mark.cpu +def test_multi_branch_without_cfg(): + dtype = torch.float32 + device = torch.device("cpu") + pipeline = _build_pipeline_on_device( + MultiBranchTestPipeline, + hidden_dim=128, + model_seed=42, + dtype=dtype, + device=device, + ) + branches_kwargs = _make_multi_branch_inputs( + n_branches=3, + batch_size=1, + channels=4, + height=16, + width=16, + dtype=dtype, + device=device, + input_seed=123, + ) + + with torch.no_grad(): + noise_pred = pipeline.predict_noise_with_multi_branch_cfg( + do_true_cfg=False, + true_cfg_scale=5.0, + branches_kwargs=branches_kwargs, + cfg_normalize=False, + ) + + assert noise_pred is not None + assert noise_pred.shape == (1, 4, 16, 16) + + +# --------------------------------------------------------------------------- +# GPU nightly: same-seed sequential vs real multi-GPU CFG parallel parity +# --------------------------------------------------------------------------- + + +def _test_cfg_parallel_worker( + local_rank: int, + world_size: int, + cfg_parallel_size: int, + dtype: torch.dtype, + test_config: dict, + result_queue: torch.multiprocessing.Queue, +): + device = torch.device(f"{current_omni_platform.device_type}:{local_rank}") + current_omni_platform.set_device(device) + _update_environment_variables( + { + "RANK": str(local_rank), + "LOCAL_RANK": str(local_rank), + "WORLD_SIZE": str(world_size), + "MASTER_ADDR": "localhost", + "MASTER_PORT": "29502", + } + ) + + init_distributed_environment() + initialize_model_parallel(cfg_parallel_size=cfg_parallel_size) + assert get_classifier_free_guidance_world_size() == cfg_parallel_size + + pipeline = _build_pipeline_on_device( + TestCFGPipeline, + hidden_dim=test_config["hidden_dim"], + model_seed=test_config["model_seed"], + dtype=dtype, + device=device, + ) + positive_kwargs, negative_kwargs = _make_two_branch_inputs( + batch_size=test_config["batch_size"], + channels=test_config["channels"], + height=test_config["height"], + width=test_config["width"], + dtype=dtype, + device=device, + input_seed=test_config["input_seed"], + ) + + with torch.no_grad(): + noise_pred = pipeline.predict_noise_maybe_with_cfg( + do_true_cfg=True, + true_cfg_scale=test_config["cfg_scale"], + positive_kwargs=positive_kwargs, + negative_kwargs=negative_kwargs, + cfg_normalize=test_config["cfg_normalize"], + kwargs=test_config["kwargs"], + ) + + assert noise_pred is not None + result_queue.put((get_classifier_free_guidance_rank(), noise_pred.cpu())) + destroy_distributed_env() + + +def _test_cfg_sequential_worker( + local_rank: int, + world_size: int, + dtype: torch.dtype, + test_config: dict, + result_queue: torch.multiprocessing.Queue, +): + device = torch.device(f"{current_omni_platform.device_type}:{local_rank}") + current_omni_platform.set_device(device) + _update_environment_variables( + { + "RANK": str(local_rank), + "LOCAL_RANK": str(local_rank), + "WORLD_SIZE": str(world_size), + "MASTER_ADDR": "localhost", + "MASTER_PORT": "29503", + } + ) + + init_distributed_environment() + initialize_model_parallel(cfg_parallel_size=1) + assert get_classifier_free_guidance_world_size() == 1 + + pipeline = _build_pipeline_on_device( + TestCFGPipeline, + hidden_dim=test_config["hidden_dim"], + model_seed=test_config["model_seed"], + dtype=dtype, + device=device, + ) + positive_kwargs, negative_kwargs = _make_two_branch_inputs( + batch_size=test_config["batch_size"], + channels=test_config["channels"], + height=test_config["height"], + width=test_config["width"], + dtype=dtype, + device=device, + input_seed=test_config["input_seed"], + ) + + with torch.no_grad(): + noise_pred = pipeline.predict_noise_maybe_with_cfg( + do_true_cfg=True, + true_cfg_scale=test_config["cfg_scale"], + positive_kwargs=positive_kwargs, + negative_kwargs=negative_kwargs, + cfg_normalize=test_config["cfg_normalize"], + kwargs=test_config["kwargs"], + ) + + assert noise_pred is not None + result_queue.put(noise_pred.cpu()) + destroy_distributed_env() + + def _test_multi_branch_parallel_worker( local_rank: int, world_size: int, @@ -494,11 +714,9 @@ def _test_multi_branch_parallel_worker( test_config: dict, result_queue: torch.multiprocessing.Queue, ): - """Worker function for multi-branch CFG parallel test.""" device = torch.device(f"{current_omni_platform.device_type}:{local_rank}") current_omni_platform.set_device(device) - - update_environment_variables( + _update_environment_variables( { "RANK": str(local_rank), "LOCAL_RANK": str(local_rank), @@ -510,33 +728,25 @@ def _test_multi_branch_parallel_worker( init_distributed_environment() initialize_model_parallel(cfg_parallel_size=cfg_parallel_size) + assert get_classifier_free_guidance_world_size() == cfg_parallel_size - cfg_rank = get_classifier_free_guidance_rank() - cfg_world_size = get_classifier_free_guidance_world_size() - assert cfg_world_size == cfg_parallel_size - - pipeline = MultiBranchTestPipeline( - in_channels=test_config["channels"], + pipeline = _build_pipeline_on_device( + MultiBranchTestPipeline, hidden_dim=test_config["hidden_dim"], - seed=test_config["model_seed"], + model_seed=test_config["model_seed"], + dtype=dtype, + device=device, + ) + branches_kwargs = _make_multi_branch_inputs( + n_branches=test_config["n_branches"], + batch_size=test_config["batch_size"], + channels=test_config["channels"], + height=test_config["height"], + width=test_config["width"], + dtype=dtype, + device=device, + input_seed=test_config["input_seed"], ) - pipeline.transformer = pipeline.transformer.to(device=device, dtype=dtype) - pipeline.transformer.eval() - - n_branches = test_config["n_branches"] - batch_size = test_config["batch_size"] - channels = test_config["channels"] - height = test_config["height"] - width = test_config["width"] - - # Create N branch inputs with distinct seeds - branches_kwargs = [] - for b in range(n_branches): - torch.manual_seed(test_config["input_seed"] + b) - if torch.cuda.is_available(): - torch.cuda.manual_seed_all(test_config["input_seed"] + b) - x = torch.randn(batch_size, channels, height, width, dtype=dtype, device=device) - branches_kwargs.append({"x": x}) with torch.no_grad(): noise_pred = pipeline.predict_noise_with_multi_branch_cfg( @@ -547,8 +757,7 @@ def _test_multi_branch_parallel_worker( ) assert noise_pred is not None - result_queue.put((cfg_rank, noise_pred.cpu())) - + result_queue.put((get_classifier_free_guidance_rank(), noise_pred.cpu())) destroy_distributed_env() @@ -559,11 +768,9 @@ def _test_multi_branch_sequential_worker( test_config: dict, result_queue: torch.multiprocessing.Queue, ): - """Worker function for sequential multi-branch CFG test (baseline).""" device = torch.device(f"{current_omni_platform.device_type}:{local_rank}") current_omni_platform.set_device(device) - - update_environment_variables( + _update_environment_variables( { "RANK": str(local_rank), "LOCAL_RANK": str(local_rank), @@ -575,31 +782,25 @@ def _test_multi_branch_sequential_worker( init_distributed_environment() initialize_model_parallel(cfg_parallel_size=1) + assert get_classifier_free_guidance_world_size() == 1 - cfg_world_size = get_classifier_free_guidance_world_size() - assert cfg_world_size == 1 - - pipeline = MultiBranchTestPipeline( - in_channels=test_config["channels"], + pipeline = _build_pipeline_on_device( + MultiBranchTestPipeline, hidden_dim=test_config["hidden_dim"], - seed=test_config["model_seed"], + model_seed=test_config["model_seed"], + dtype=dtype, + device=device, + ) + branches_kwargs = _make_multi_branch_inputs( + n_branches=test_config["n_branches"], + batch_size=test_config["batch_size"], + channels=test_config["channels"], + height=test_config["height"], + width=test_config["width"], + dtype=dtype, + device=device, + input_seed=test_config["input_seed"], ) - pipeline.transformer = pipeline.transformer.to(device=device, dtype=dtype) - pipeline.transformer.eval() - - n_branches = test_config["n_branches"] - batch_size = test_config["batch_size"] - channels = test_config["channels"] - height = test_config["height"] - width = test_config["width"] - - branches_kwargs = [] - for b in range(n_branches): - torch.manual_seed(test_config["input_seed"] + b) - if torch.cuda.is_available(): - torch.cuda.manual_seed_all(test_config["input_seed"] + b) - x = torch.randn(batch_size, channels, height, width, dtype=dtype, device=device) - branches_kwargs.append({"x": x}) with torch.no_grad(): noise_pred = pipeline.predict_noise_with_multi_branch_cfg( @@ -611,58 +812,86 @@ def _test_multi_branch_sequential_worker( assert noise_pred is not None result_queue.put(noise_pred.cpu()) - destroy_distributed_env() -@pytest.mark.parametrize( - "cfg_parallel_size,n_branches", - [ - (2, 2), # 2 branches on 2 GPUs: [[0],[1]] - (2, 3), # 3 branches on 2 GPUs: [[0,2],[1]] - (3, 3), # 3 branches on 3 GPUs: [[0],[1],[2]] - (2, 4), # 4 branches on 2 GPUs: [[0,2],[1,3]] - ], -) -@pytest.mark.parametrize("dtype", [torch.bfloat16]) -@pytest.mark.parametrize("batch_size", [2]) -@pytest.mark.parametrize("cfg_normalize", [False, True]) -def test_predict_noise_with_multi_branch_cfg( +def _run_two_branch_gpu_parity( + *, cfg_parallel_size: int, - n_branches: int, dtype: torch.dtype, batch_size: int, cfg_normalize: bool, -): - """ - Test that predict_noise_with_multi_branch_cfg produces identical results - with and without CFG parallel for N-branch models. - - Args: - cfg_parallel_size: Number of GPUs for CFG parallel - n_branches: Number of CFG branches - dtype: Data type for computation - batch_size: Batch size for testing - cfg_normalize: Whether to normalize CFG output - """ - available_gpus = current_omni_platform.get_device_count() - if available_gpus < cfg_parallel_size: - pytest.skip(f"Test requires {cfg_parallel_size} GPUs but only {available_gpus} available") - - if n_branches == 2: - cfg_scale = 5.0 - elif n_branches == 3: - cfg_scale = {"text": 5.0, "image": 2.0} - else: - cfg_scale = {"text": 5.0, "image": 2.0, "vid_ref": 1.5} - + extra_kwargs: dict[str, Any] | None, +) -> None: test_config = { "batch_size": batch_size, "channels": 4, "height": 16, "width": 16, "hidden_dim": 128, - "cfg_scale": cfg_scale, + "cfg_scale": 7.5, + "cfg_normalize": cfg_normalize, + "model_seed": 42, + "input_seed": 123, + "kwargs": extra_kwargs, + } + + mp_context = torch.multiprocessing.get_context("spawn") + manager = mp_context.Manager() + baseline_queue = manager.Queue() + cfg_parallel_queue = manager.Queue() + + torch.multiprocessing.spawn( + _test_cfg_sequential_worker, + args=(1, dtype, test_config, baseline_queue), + nprocs=1, + ) + torch.multiprocessing.spawn( + _test_cfg_parallel_worker, + args=(cfg_parallel_size, cfg_parallel_size, dtype, test_config, cfg_parallel_queue), + nprocs=cfg_parallel_size, + ) + + baseline_output = baseline_queue.get() + cfg_parallel_outputs = [cfg_parallel_queue.get() for _ in range(cfg_parallel_size)] + cfg_parallel_outputs.sort(key=lambda item: item[0]) + cfg_parallel_output = cfg_parallel_outputs[0][1] + + for cfg_rank, rank_output in cfg_parallel_outputs[1:]: + torch.testing.assert_close( + rank_output, + cfg_parallel_output, + rtol=0, + atol=0, + msg=f"CFG parallel ranks differ (rank 0 vs rank {cfg_rank})", + ) + + _assert_cfg_outputs_close( + cfg_parallel_output, + baseline_output, + dtype=dtype, + msg=( + f"GPU CFG parallel output differs from sequential CFG " + f"(cfg_parallel_size={cfg_parallel_size}, cfg_normalize={cfg_normalize})" + ), + ) + + +def _run_multi_branch_gpu_parity( + *, + cfg_parallel_size: int, + n_branches: int, + dtype: torch.dtype, + batch_size: int, + cfg_normalize: bool, +) -> None: + test_config = { + "batch_size": batch_size, + "channels": 4, + "height": 16, + "width": 16, + "hidden_dim": 128, + "cfg_scale": _multi_branch_cfg_scale(n_branches), "cfg_normalize": cfg_normalize, "model_seed": 42, "input_seed": 123, @@ -674,14 +903,11 @@ def test_predict_noise_with_multi_branch_cfg( baseline_queue = manager.Queue() cfg_parallel_queue = manager.Queue() - # Run baseline (sequential, cfgp=1) torch.multiprocessing.spawn( _test_multi_branch_sequential_worker, args=(1, dtype, test_config, baseline_queue), nprocs=1, ) - - # Run CFG parallel torch.multiprocessing.spawn( _test_multi_branch_parallel_worker, args=(cfg_parallel_size, cfg_parallel_size, dtype, test_config, cfg_parallel_queue), @@ -693,7 +919,6 @@ def test_predict_noise_with_multi_branch_cfg( cfg_parallel_outputs.sort(key=lambda item: item[0]) cfg_parallel_output = cfg_parallel_outputs[0][1] - # All ranks should produce identical output for cfg_rank, rank_output in cfg_parallel_outputs[1:]: torch.testing.assert_close( rank_output, @@ -703,73 +928,62 @@ def test_predict_noise_with_multi_branch_cfg( msg=f"Multi-branch CFG parallel ranks differ (rank 0 vs rank {cfg_rank})", ) - assert baseline_output.shape == cfg_parallel_output.shape, ( - f"Shape mismatch: baseline {baseline_output.shape} vs CFG parallel {cfg_parallel_output.shape}" - ) - - if dtype == torch.float32: - rtol, atol = 1e-5, 1e-5 - elif dtype == torch.bfloat16: - rtol, atol = 1e-2, 1e-2 - else: - rtol, atol = 1e-3, 1e-3 - - torch.testing.assert_close( + _assert_cfg_outputs_close( cfg_parallel_output, baseline_output, - rtol=rtol, - atol=atol, + dtype=dtype, msg=( - f"Multi-branch CFG parallel output differs from sequential\n" - f" n_branches={n_branches}, cfg_parallel_size={cfg_parallel_size}\n" - f" dtype={dtype}, cfg_normalize={cfg_normalize}\n" - f" Max diff: {(cfg_parallel_output - baseline_output).abs().max().item():.6e}" + f"GPU multi-branch CFG parallel differs from sequential " + f"(n_branches={n_branches}, cfg_parallel_size={cfg_parallel_size})" ), ) - print( - f"✓ Test passed: multi_branch n_branches={n_branches}, " - f"cfg_size={cfg_parallel_size}, dtype={dtype}, cfg_normalize={cfg_normalize}" + +@pytest.mark.full_model +@pytest.mark.diffusion +@pytest.mark.parallel +@hardware_test(res={"cuda": "L4"}, num_cards=2) +@pytest.mark.parametrize("batch_size", [2]) +@pytest.mark.parametrize("cfg_normalize", [False, True]) +@pytest.mark.parametrize("extra_kwargs", [None, {"step_i": 1}]) +def test_predict_noise_with_cfg_parity( + batch_size: int, + cfg_normalize: bool, + extra_kwargs: dict[str, Any] | None, +): + _run_two_branch_gpu_parity( + cfg_parallel_size=2, + dtype=torch.bfloat16, + batch_size=batch_size, + cfg_normalize=cfg_normalize, + extra_kwargs=extra_kwargs, ) -@pytest.mark.parametrize("dtype", [torch.bfloat16]) -def test_multi_branch_without_cfg(dtype: torch.dtype): - """ - Test predict_noise_with_multi_branch_cfg when do_true_cfg=False. - - When CFG is disabled, only the first branch (positive) should be computed. - This test runs on a single GPU without distributed environment. - """ - available_gpus = current_omni_platform.get_device_count() - if available_gpus < 1: - pytest.skip("Test requires at least 1 GPU") - - device = torch.device(f"{current_omni_platform.device_type}:0") - current_omni_platform.set_device(device) - - pipeline = MultiBranchTestPipeline(in_channels=4, hidden_dim=128, seed=42) - pipeline.transformer = pipeline.transformer.to(device=device, dtype=dtype) - pipeline.transformer.eval() - - # Create 3 branch inputs (only first should be used) - branches_kwargs = [] - for b in range(3): - torch.manual_seed(123 + b) - if torch.cuda.is_available(): - torch.cuda.manual_seed_all(123 + b) - x = torch.randn(1, 4, 16, 16, dtype=dtype, device=device) - branches_kwargs.append({"x": x}) - - with torch.no_grad(): - noise_pred = pipeline.predict_noise_with_multi_branch_cfg( - do_true_cfg=False, # No CFG - true_cfg_scale=5.0, - branches_kwargs=branches_kwargs, - cfg_normalize=False, - ) - - assert noise_pred is not None - assert noise_pred.shape == (1, 4, 16, 16) - - print(f"✓ Test passed: multi_branch predict_noise without CFG (dtype={dtype})") +@pytest.mark.full_model +@pytest.mark.diffusion +@pytest.mark.parallel +@pytest.mark.parametrize( + "cfg_parallel_size,n_branches", + [ + pytest.param(2, 2, marks=_L4_TWO_GPU), + pytest.param(2, 3, marks=_L4_TWO_GPU), + pytest.param(3, 3, marks=_L4_THREE_GPU), + pytest.param(2, 4, marks=_L4_TWO_GPU), + ], +) +@pytest.mark.parametrize("batch_size", [2]) +@pytest.mark.parametrize("cfg_normalize", [False, True]) +def test_predict_noise_with_multi_branch_cfg_parity( + cfg_parallel_size: int, + n_branches: int, + batch_size: int, + cfg_normalize: bool, +): + _run_multi_branch_gpu_parity( + cfg_parallel_size=cfg_parallel_size, + n_branches=n_branches, + dtype=torch.bfloat16, + batch_size=batch_size, + cfg_normalize=cfg_normalize, + ) diff --git a/tests/diffusion/distributed/test_comm.py b/tests/diffusion/distributed/test_comm.py index 3e408c10a..647389048 100644 --- a/tests/diffusion/distributed/test_comm.py +++ b/tests/diffusion/distributed/test_comm.py @@ -1,12 +1,20 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for SeqAllToAll4D and SeqAllToAll5D communication primitives.""" +"""Tests for SeqAllToAll4D, SeqAllToAll5D, and RingComm communication primitives. + +CPU tests use gloo on CPU tensors (no GPU). Nightly parity tests run the same +checks on real multi-GPU NCCL collectives. +""" + +from __future__ import annotations import os +from typing import Literal import pytest import torch +from tests.helpers.mark import hardware_marks from vllm_omni.diffusion.distributed.comm import RingComm, SeqAllToAll4D, SeqAllToAll5D from vllm_omni.diffusion.distributed.parallel_state import ( destroy_distributed_env, @@ -16,56 +24,51 @@ from vllm_omni.diffusion.distributed.parallel_state import ( ) from vllm_omni.platforms import current_omni_platform +_L4_TWO_GPU = hardware_marks(res={"cuda": "L4"}, num_cards=2) +_L4_FOUR_GPU = hardware_marks(res={"cuda": "L4"}, num_cards=4) -def update_environment_variables(envs_dict: dict[str, str]): - """Update multiple environment variables with logging.""" - for k, v in envs_dict.items(): - os.environ[k] = v +DeviceKind = Literal["cpu", "cuda"] -@pytest.mark.parametrize("world_size", [2, 4]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -@pytest.mark.parametrize("batch_size", [2]) -@pytest.mark.parametrize("seq_len_per_rank", [8]) -@pytest.mark.parametrize("num_heads", [8]) -@pytest.mark.parametrize("head_size", [32]) -@pytest.mark.parametrize("use_sync", [False, True]) -def test_4d_identity( +def _update_environment_variables(envs_dict: dict[str, str]) -> None: + for key, value in envs_dict.items(): + os.environ[key] = value + + +def _worker_device(local_rank: int, device_kind: DeviceKind) -> torch.device: + if device_kind == "cpu": + return torch.device("cpu") + return torch.device(f"{current_omni_platform.device_type}:{local_rank}") + + +def _init_worker( + local_rank: int, world_size: int, - dtype: torch.dtype, - batch_size: int, - seq_len_per_rank: int, - num_heads: int, - head_size: int, - use_sync: bool, -): - """Test that two consecutive all-to-all operations return the original input.""" - # Skip if not enough GPUs available - available_gpus = current_omni_platform.get_device_count() - if available_gpus < world_size: - pytest.skip(f"Test requires {world_size} GPUs but only {available_gpus} available") - - # Ensure num_heads is divisible by world_size - if num_heads % world_size != 0: - pytest.skip(f"num_heads ({num_heads}) not divisible by world_size ({world_size})") - - # Run test with multiprocessing spawn - torch.multiprocessing.spawn( - _test_4d_identity_worker, - args=( - world_size, - dtype, - batch_size, - seq_len_per_rank, - num_heads, - head_size, - use_sync, - ), - nprocs=world_size, + master_port: int, + device_kind: DeviceKind, +) -> None: + _update_environment_variables( + { + "RANK": str(local_rank), + "LOCAL_RANK": str(local_rank), + "WORLD_SIZE": str(world_size), + "MASTER_ADDR": "localhost", + "MASTER_PORT": str(master_port), + } ) + backend = "gloo" if device_kind == "cpu" else None + init_distributed_environment(backend=backend) -def _test_4d_identity_worker( +def _close_tolerance(dtype: torch.dtype) -> tuple[float, float]: + if dtype == torch.bfloat16: + return 1e-3, 1e-3 + if dtype == torch.float16: + return 1e-3, 1e-3 + return 1e-5, 1e-5 + + +def _run_4d_identity( local_rank: int, world_size: int, dtype: torch.dtype, @@ -74,29 +77,17 @@ def _test_4d_identity_worker( num_heads: int, head_size: int, use_sync: bool, -): - """Worker function for test_4d_identity.""" - # Set device - device = torch.device(f"{current_omni_platform.device_type}:{local_rank}") - current_omni_platform.set_device(device) + device_kind: DeviceKind, + master_port: int, +) -> None: + device = _worker_device(local_rank, device_kind) + if device_kind == "cuda": + current_omni_platform.set_device(device) - # Set environment variables for distributed training - update_environment_variables( - { - "RANK": str(local_rank), - "LOCAL_RANK": str(local_rank), - "WORLD_SIZE": str(world_size), - "MASTER_ADDR": "localhost", - "MASTER_PORT": "29500", - } - ) + _init_worker(local_rank, world_size, master_port, device_kind) + initialize_model_parallel(ulysses_degree=world_size) + sp_group = get_sp_group().ulysses_group - # Initialize distributed environment - init_distributed_environment() - initialize_model_parallel(ulysses_degree=world_size) # test ulysses sp by default - sp_group = get_sp_group().ulysses_group # get ulysses sp group not ring sp group - - # Create input tensor: (bs, seqlen/P, hc, hs) torch.manual_seed(42 + local_rank) input_tensor = torch.randn( batch_size, @@ -106,100 +97,36 @@ def _test_4d_identity_worker( dtype=dtype, device=device, ) - - # Save original input for comparison original_input = input_tensor.clone() - # First all-to-all: (bs, seqlen/P, hc, hs) -> (bs, seqlen, hc/P, hs) - intermediate = SeqAllToAll4D.apply( - sp_group, - input_tensor, - 2, # scatter head dimension - 1, # gather sequence dimension - use_sync, - ) - - # Verify intermediate shape - expected_shape = ( + intermediate = SeqAllToAll4D.apply(sp_group, input_tensor, 2, 1, use_sync) + expected_intermediate_shape = ( batch_size, seq_len_per_rank * world_size, num_heads // world_size, head_size, ) - assert intermediate.shape == expected_shape, ( - f"Intermediate shape mismatch: expected {expected_shape}, got {intermediate.shape}" + assert intermediate.shape == expected_intermediate_shape, ( + f"Intermediate shape mismatch: expected {expected_intermediate_shape}, got {intermediate.shape}" ) - # Second all-to-all: (bs, seqlen, hc/P, hs) -> (bs, seqlen/P, hc, hs) - output = SeqAllToAll4D.apply( - sp_group, - intermediate, - 1, # scatter sequence dimension - 2, # gather head dimension - use_sync, - ) - - # Verify output shape matches input + output = SeqAllToAll4D.apply(sp_group, intermediate, 1, 2, use_sync) assert output.shape == original_input.shape, ( f"Output shape mismatch: expected {original_input.shape}, got {output.shape}" ) - # Verify output matches original input + rtol, atol = _close_tolerance(dtype) torch.testing.assert_close( output, original_input, - rtol=1e-5, - atol=1e-5, + rtol=rtol, + atol=atol, msg="Output does not match original input after two all-to-all operations", ) - - # Cleanup distributed environment destroy_distributed_env() -@pytest.mark.parametrize("world_size", [2, 4]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -@pytest.mark.parametrize("batch_size", [2]) -@pytest.mark.parametrize("seq_len_per_rank", [8]) -@pytest.mark.parametrize("num_heads", [8]) -@pytest.mark.parametrize("head_size", [32]) -@pytest.mark.parametrize("use_sync", [False, True]) -def test_5d_identity( - world_size: int, - dtype: torch.dtype, - batch_size: int, - seq_len_per_rank: int, - num_heads: int, - head_size: int, - use_sync: bool, -): - """Test that two consecutive all-to-all operations return the original input.""" - # Skip if not enough GPUs available - available_gpus = current_omni_platform.get_device_count() - if available_gpus < world_size: - pytest.skip(f"Test requires {world_size} GPUs but only {available_gpus} available") - - # Ensure num_heads is divisible by world_size - if num_heads % world_size != 0: - pytest.skip(f"num_heads ({num_heads}) not divisible by world_size ({world_size})") - - # Run test with multiprocessing spawn - torch.multiprocessing.spawn( - _test_5d_identity_worker, - args=( - world_size, - dtype, - batch_size, - seq_len_per_rank, - num_heads, - head_size, - use_sync, - ), - nprocs=world_size, - ) - - -def _test_5d_identity_worker( +def _run_5d_identity( local_rank: int, world_size: int, dtype: torch.dtype, @@ -208,199 +135,329 @@ def _test_5d_identity_worker( num_heads: int, head_size: int, use_sync: bool, -): - """Worker function for test_5d_identity.""" - # Set device - device = torch.device(f"{current_omni_platform.device_type}:{local_rank}") - current_omni_platform.set_device(device) + device_kind: DeviceKind, + master_port: int, +) -> None: + device = _worker_device(local_rank, device_kind) + if device_kind == "cuda": + current_omni_platform.set_device(device) - # Set environment variables for distributed training - update_environment_variables( - { - "RANK": str(local_rank), - "LOCAL_RANK": str(local_rank), - "WORLD_SIZE": str(world_size), - "MASTER_ADDR": "localhost", - "MASTER_PORT": "29500", - } - ) + _init_worker(local_rank, world_size, master_port, device_kind) + initialize_model_parallel(ulysses_degree=world_size) + sp_group = get_sp_group().ulysses_group - # Initialize distributed environment - init_distributed_environment() - initialize_model_parallel(ulysses_degree=world_size) # test ulysses sp by default - sp_group = get_sp_group().ulysses_group # get ulysses sp group not ring sp group - - # Create input tensor: (bs, seqlen/P, 3, hc, hs) - # The '3' dimension is for Q, K, V torch.manual_seed(42 + local_rank) input_tensor = torch.randn( batch_size, seq_len_per_rank, - 3, # Q, K, V + 3, num_heads, head_size, dtype=dtype, device=device, ) - - # Save original input for comparison original_input = input_tensor.clone() - # First all-to-all: (bs, seqlen/P, 3, hc, hs) -> (bs, seqlen, 3, hc/P, hs) - intermediate = SeqAllToAll5D.apply( - sp_group, - input_tensor, - 3, # scatter head dimension - 1, # gather sequence dimension - use_sync, - ) - - # Verify intermediate shape - expected_shape = ( + intermediate = SeqAllToAll5D.apply(sp_group, input_tensor, 3, 1, use_sync) + expected_intermediate_shape = ( batch_size, seq_len_per_rank * world_size, 3, num_heads // world_size, head_size, ) - assert intermediate.shape == expected_shape, ( - f"Intermediate shape mismatch: expected {expected_shape}, got {intermediate.shape}" + assert intermediate.shape == expected_intermediate_shape, ( + f"Intermediate shape mismatch: expected {expected_intermediate_shape}, got {intermediate.shape}" ) - # Second all-to-all: (bs, seqlen, 3, hc/P, hs) -> (bs, seqlen/P, 3, hc, hs) - output = SeqAllToAll5D.apply( - sp_group, - intermediate, - 1, # scatter sequence dimension - 3, # gather head dimension - use_sync, - ) - - # Verify output shape matches input + output = SeqAllToAll5D.apply(sp_group, intermediate, 1, 3, use_sync) assert output.shape == original_input.shape, ( f"Output shape mismatch: expected {original_input.shape}, got {output.shape}" ) - # Verify output matches original input + rtol, atol = _close_tolerance(dtype) torch.testing.assert_close( output, original_input, - rtol=1e-5, - atol=1e-5, + rtol=rtol, + atol=atol, msg="Output does not match original input after two all-to-all operations", ) - - # Cleanup distributed environment destroy_distributed_env() -@pytest.mark.parametrize("world_size", [2, 4]) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -@pytest.mark.parametrize("batch_size", [2]) -@pytest.mark.parametrize("num_heads", [8]) -@pytest.mark.parametrize("head_size", [128]) -def test_ring_p2p( - world_size: int, - dtype: torch.dtype, - batch_size: int, - num_heads: int, - head_size: int, -): - """Test Ring P2P communication (send_recv).""" - # Skip if not enough GPUs available - available_gpus = current_omni_platform.get_device_count() - if available_gpus < world_size: - pytest.skip(f"Test requires {world_size} GPUs but only {available_gpus} available") - - torch.multiprocessing.spawn( - _test_ring_p2p_worker, - args=(world_size, dtype, batch_size, num_heads, head_size), - nprocs=world_size, - ) - - -def _test_ring_p2p_worker( +def _run_ring_p2p( local_rank: int, world_size: int, dtype: torch.dtype, batch_size: int, num_heads: int, head_size: int, -): - """Worker for Ring P2P test.""" - import sys + device_kind: DeviceKind, + master_port: int, +) -> None: + device = _worker_device(local_rank, device_kind) + if device_kind == "cuda": + current_omni_platform.set_device(device) - # Set device - device = torch.device(f"{current_omni_platform.device_type}:{local_rank}") - current_omni_platform.set_device(device) - - # Set env vars - # Use a different port to avoid conflict with other tests if run in parallel - update_environment_variables( - { - "RANK": str(local_rank), - "LOCAL_RANK": str(local_rank), - "WORLD_SIZE": str(world_size), - "MASTER_ADDR": "localhost", - "MASTER_PORT": "29501", - } - ) - - # Init distributed try: - init_distributed_environment() - # Ring degree = world_size to test ring group + _init_worker(local_rank, world_size, master_port, device_kind) initialize_model_parallel(ring_degree=world_size) sp_group = get_sp_group() - - print(f"[Rank {local_rank}] Initialized. Ring group size: {sp_group.ring_group.size()}") - sys.stdout.flush() - - # Create RingComm comm = RingComm(sp_group.ring_group) - # Create tensor: rank-specific data - # (batch, num_heads, head_size) - # Fill with rank value + 1 to avoid 0 and make verification easy input_tensor = torch.full( - (batch_size, num_heads, head_size), fill_value=float(local_rank + 1), dtype=dtype, device=device + (batch_size, num_heads, head_size), + fill_value=float(local_rank + 1), + dtype=dtype, + device=device, ) - - print(f"[Rank {local_rank}] Input sum: {input_tensor.sum().item()}") - sys.stdout.flush() - - # Send input, receive from prev - # RingComm.send_recv sends to next, receives from prev - t0 = __import__("time").time() recv_tensor = comm.send_recv(input_tensor) comm.commit() comm.wait() - t1 = __import__("time").time() - print(f"[Rank {local_rank}] Communication done in {t1 - t0:.4f}s") - - # Verify - # Expected value: from (rank - 1) % world_size prev_rank = (local_rank - 1 + world_size) % world_size expected_value = float(prev_rank + 1) - - recv_sum = recv_tensor.sum().item() - print(f"[Rank {local_rank}] Received sum: {recv_sum}, Expected value: {expected_value}") - sys.stdout.flush() - expected_tensor = torch.full_like(recv_tensor, fill_value=expected_value) - - # Use a slightly loose tolerance for bfloat16 + rtol, atol = _close_tolerance(dtype) torch.testing.assert_close( - recv_tensor, expected_tensor, rtol=1e-3, atol=1e-3, msg=f"[Rank {local_rank}] Data mismatch!" + recv_tensor, + expected_tensor, + rtol=rtol, + atol=atol, + msg=f"[Rank {local_rank}] Ring P2P data mismatch", ) - print(f"[Rank {local_rank}] Verification PASSED") - - except Exception as e: - print(f"[Rank {local_rank}] FAILED with error: {e}") - import traceback - - traceback.print_exc() - raise e finally: destroy_distributed_env() + + +def _spawn_4d_identity( + *, + world_size: int, + dtype: torch.dtype, + batch_size: int, + seq_len_per_rank: int, + num_heads: int, + head_size: int, + use_sync: bool, + device_kind: DeviceKind, + master_port: int, +) -> None: + torch.multiprocessing.spawn( + _run_4d_identity, + args=( + world_size, + dtype, + batch_size, + seq_len_per_rank, + num_heads, + head_size, + use_sync, + device_kind, + master_port, + ), + nprocs=world_size, + ) + + +def _spawn_5d_identity( + *, + world_size: int, + dtype: torch.dtype, + batch_size: int, + seq_len_per_rank: int, + num_heads: int, + head_size: int, + use_sync: bool, + device_kind: DeviceKind, + master_port: int, +) -> None: + torch.multiprocessing.spawn( + _run_5d_identity, + args=( + world_size, + dtype, + batch_size, + seq_len_per_rank, + num_heads, + head_size, + use_sync, + device_kind, + master_port, + ), + nprocs=world_size, + ) + + +def _spawn_ring_p2p( + *, + world_size: int, + dtype: torch.dtype, + batch_size: int, + num_heads: int, + head_size: int, + device_kind: DeviceKind, + master_port: int, +) -> None: + torch.multiprocessing.spawn( + _run_ring_p2p, + args=(world_size, dtype, batch_size, num_heads, head_size, device_kind, master_port), + nprocs=world_size, + ) + + +def _require_heads_divisible(num_heads: int, world_size: int) -> None: + if num_heads % world_size != 0: + pytest.skip(f"num_heads ({num_heads}) not divisible by world_size ({world_size})") + + +def _require_gpus(world_size: int) -> None: + available_gpus = current_omni_platform.get_device_count() + if available_gpus < world_size: + pytest.skip(f"Test requires {world_size} GPUs but only {available_gpus} available") + + +# --------------------------------------------------------------------------- +# CPU: gloo collectives on CPU tensors (no GPU) +# --------------------------------------------------------------------------- + + +@pytest.mark.core_model +@pytest.mark.diffusion +@pytest.mark.cpu +@pytest.mark.parametrize("world_size", [2, 4]) +@pytest.mark.parametrize("use_sync", [False, True]) +def test_4d_identity(world_size: int, use_sync: bool): + _require_heads_divisible(8, world_size) + _spawn_4d_identity( + world_size=world_size, + dtype=torch.float32, + batch_size=2, + seq_len_per_rank=8, + num_heads=8, + head_size=32, + use_sync=use_sync, + device_kind="cpu", + master_port=29610, + ) + + +@pytest.mark.core_model +@pytest.mark.diffusion +@pytest.mark.cpu +@pytest.mark.parametrize("world_size", [2, 4]) +@pytest.mark.parametrize("use_sync", [False, True]) +def test_5d_identity(world_size: int, use_sync: bool): + _require_heads_divisible(8, world_size) + _spawn_5d_identity( + world_size=world_size, + dtype=torch.float32, + batch_size=2, + seq_len_per_rank=8, + num_heads=8, + head_size=32, + use_sync=use_sync, + device_kind="cpu", + master_port=29611, + ) + + +@pytest.mark.core_model +@pytest.mark.diffusion +@pytest.mark.cpu +@pytest.mark.parametrize("world_size", [2, 4]) +def test_ring_p2p(world_size: int): + _spawn_ring_p2p( + world_size=world_size, + dtype=torch.float32, + batch_size=2, + num_heads=8, + head_size=128, + device_kind="cpu", + master_port=29612, + ) + + +# --------------------------------------------------------------------------- +# Nightly: same checks on real multi-GPU NCCL collectives +# --------------------------------------------------------------------------- + + +@pytest.mark.full_model +@pytest.mark.diffusion +@pytest.mark.parallel +@pytest.mark.parametrize( + "world_size", + [ + pytest.param(2, marks=_L4_TWO_GPU), + pytest.param(4, marks=_L4_FOUR_GPU), + ], +) +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("use_sync", [False, True]) +def test_4d_identity_parity(world_size: int, dtype: torch.dtype, use_sync: bool): + _require_gpus(world_size) + _require_heads_divisible(8, world_size) + _spawn_4d_identity( + world_size=world_size, + dtype=dtype, + batch_size=2, + seq_len_per_rank=8, + num_heads=8, + head_size=32, + use_sync=use_sync, + device_kind="cuda", + master_port=29500, + ) + + +@pytest.mark.full_model +@pytest.mark.diffusion +@pytest.mark.parallel +@pytest.mark.parametrize( + "world_size", + [ + pytest.param(2, marks=_L4_TWO_GPU), + pytest.param(4, marks=_L4_FOUR_GPU), + ], +) +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("use_sync", [False, True]) +def test_5d_identity_parity(world_size: int, dtype: torch.dtype, use_sync: bool): + _require_gpus(world_size) + _require_heads_divisible(8, world_size) + _spawn_5d_identity( + world_size=world_size, + dtype=dtype, + batch_size=2, + seq_len_per_rank=8, + num_heads=8, + head_size=32, + use_sync=use_sync, + device_kind="cuda", + master_port=29502, + ) + + +@pytest.mark.full_model +@pytest.mark.diffusion +@pytest.mark.parallel +@pytest.mark.parametrize( + "world_size", + [ + pytest.param(2, marks=_L4_TWO_GPU), + pytest.param(4, marks=_L4_FOUR_GPU), + ], +) +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +def test_ring_p2p_parity(world_size: int, dtype: torch.dtype): + _require_gpus(world_size) + _spawn_ring_p2p( + world_size=world_size, + dtype=dtype, + batch_size=2, + num_heads=8, + head_size=128, + device_kind="cuda", + master_port=29501, + ) diff --git a/tests/diffusion/distributed/test_pipeline_parallel.py b/tests/diffusion/distributed/test_pipeline_parallel.py index b76bf8c89..0b48ed266 100644 --- a/tests/diffusion/distributed/test_pipeline_parallel.py +++ b/tests/diffusion/distributed/test_pipeline_parallel.py @@ -1,8 +1,17 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for PipelineParallelMixin and related PP communication helpers. + +CPU unit tests exercise mixin logic without a process group. Distributed CPU +tests use gloo on CPU tensors; nightly parity tests run the same checks on +real multi-GPU NCCL collectives. +""" + +from __future__ import annotations import os import socket +from typing import Literal import pytest import torch @@ -11,6 +20,7 @@ from vllm.sequence import IntermediateTensors from vllm.v1.worker.gpu_worker import AsyncIntermediateTensors import vllm_omni.diffusion.distributed.pipeline_parallel as pp_module +from tests.helpers.mark import hardware_marks from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin from vllm_omni.diffusion.distributed.parallel_state import ( destroy_distributed_env, @@ -22,7 +32,12 @@ from vllm_omni.diffusion.distributed.parallel_state import ( from vllm_omni.diffusion.distributed.pipeline_parallel import AsyncLatents, PipelineParallelMixin from vllm_omni.platforms import current_omni_platform -pytestmark = [pytest.mark.parallel] +_L4_TWO_GPU = hardware_marks(res={"cuda": "L4"}, num_cards=2) +_L4_THREE_GPU = hardware_marks(res={"cuda": "L4"}, num_cards=3) +_L4_FOUR_GPU = hardware_marks(res={"cuda": "L4"}, num_cards=4) + +DeviceKind = Literal["cpu", "cuda"] +_UNIT_MARKS = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] def _find_free_port() -> str: @@ -36,6 +51,22 @@ def update_environment_variables(envs_dict: dict[str, str]) -> None: os.environ[k] = v +def _cleanup_distributed() -> None: + """Destroy omni distributed state and clear test-process leftovers. + + ``destroy_distributed_env`` leaves ``vllm.distributed.parallel_state._PP`` + aliased to the destroyed group. Parent-process baselines must clear that + dangling reference (and RANK/MASTER_* env) so later unit tests that call + ``vllm.distributed.initialize_model_parallel`` do not see stale state. + """ + destroy_distributed_env() + import vllm.distributed.parallel_state as vllm_parallel_state + + vllm_parallel_state._PP = None + for key in ("MASTER_ADDR", "MASTER_PORT", "RANK", "WORLD_SIZE", "LOCAL_RANK"): + os.environ.pop(key, None) + + # --------------------------------------------------------------------------- # Shared stubs used by both unit and distributed tests # --------------------------------------------------------------------------- @@ -143,7 +174,7 @@ class MockPipelineParallel(PipelineParallelMixin, CFGParallelMixin): class TestAsyncLatents: """Verifies the lazy-resolution behaviour of AsyncLatents without a real process group.""" - pytestmark = [pytest.mark.cpu] + pytestmark = _UNIT_MARKS def _make(self, tensor: torch.Tensor, handles: list | None = None, postproc: list | None = None) -> AsyncLatents: return AsyncLatents({"latents": tensor}, handles or [], postproc or []) @@ -214,7 +245,7 @@ class TestAsyncLatents: class TestSyncPPSend: """Verifies PipelineParallelMixin's internal PP-send flush.""" - pytestmark = [pytest.mark.cpu] + pytestmark = _UNIT_MARKS @staticmethod def _make_pipeline() -> PipelineParallelMixin: @@ -247,7 +278,7 @@ class TestSyncPPSend: class TestDiffuseWrapper: """Verifies that PipelineParallelMixin flushes pending sends when diffuse() exits.""" - pytestmark = [pytest.mark.cpu] + pytestmark = _UNIT_MARKS def test_diffuse_flushes_pending_sends_on_success(self): work = FakeWork() @@ -289,7 +320,7 @@ class TestDiffuseWrapper: class TestVaeDecodeGuard: - pytestmark = [pytest.mark.cpu] + pytestmark = _UNIT_MARKS @staticmethod def _make_pipeline(distributed_enabled: bool = False) -> PipelineParallelMixin: @@ -351,6 +382,8 @@ class TestVaeDecodeGuard: assert pipeline.vae.decode.__doc__ == "Original decode docstring." +@pytest.mark.core_model +@pytest.mark.diffusion @pytest.mark.cpu def test_pipeline_parallel_requires_cfg_mixin(): with pytest.raises(TypeError, match="inherits PipelineParallelMixin but not CFGParallelMixin"): @@ -359,6 +392,8 @@ def test_pipeline_parallel_requires_cfg_mixin(): pass +@pytest.mark.core_model +@pytest.mark.diffusion @pytest.mark.cpu def test_pipeline_parallel_requires_mro_before_cfg_mixin(): with pytest.raises(TypeError, match="must inherit PipelineParallelMixin before CFGParallelMixin"): @@ -372,10 +407,22 @@ def test_pipeline_parallel_requires_mro_before_cfg_mixin(): # --------------------------------------------------------------------------- -def init_dist(local_rank: int, world_size: int, master_port: str) -> torch.device: +def _worker_device(local_rank: int, device_kind: DeviceKind) -> torch.device: + if device_kind == "cpu": + return torch.device("cpu") + return torch.device(f"{current_omni_platform.device_type}:{local_rank}") + + +def init_dist( + local_rank: int, + world_size: int, + master_port: str, + device_kind: DeviceKind, +) -> torch.device: """Initialise the distributed environment for a spawned worker.""" - device = torch.device(f"{current_omni_platform.device_type}:{local_rank}") - current_omni_platform.set_device(device) + device = _worker_device(local_rank, device_kind) + if device_kind == "cuda": + current_omni_platform.set_device(device) update_environment_variables( { "RANK": str(local_rank), @@ -385,13 +432,31 @@ def init_dist(local_rank: int, world_size: int, master_port: str) -> torch.devic "MASTER_PORT": master_port, } ) - init_distributed_environment() + backend = "gloo" if device_kind == "cpu" else None + init_distributed_environment(backend=backend) return device +def _mp_backend(device_kind: DeviceKind) -> str | None: + """Backend for model-parallel process groups. + + ``GroupCoordinator.all_gather`` always uses ``device_group``. CPU tensors + therefore require gloo device groups; otherwise a CUDA host's default NCCL + backend breaks PP+CFG CPU tests at CFG all_gather (pp2-cfg2). + isend/irecv of CPU tensors can already fall back to ``cpu_group``. + """ + return "gloo" if device_kind == "cpu" else None + + +def _seed_device(input_seed: int, device: torch.device) -> None: + torch.manual_seed(input_seed) + if device.type == "cuda": + torch.cuda.manual_seed_all(input_seed) + + def make_pipeline_and_inputs( test_config: dict, dtype: torch.dtype, device: torch.device, do_true_cfg: bool = False -) -> tuple["MockPipelineParallel", dict, dict | None]: +) -> tuple[MockPipelineParallel, dict, dict | None]: """Create a MockPipelineParallel and seeded inputs from a test_config dict. Must be called after ``initialize_model_parallel`` so that ``make_layers`` @@ -408,16 +473,12 @@ def make_pipeline_and_inputs( dtype=dtype, ) - torch.manual_seed(test_config["input_seed"]) - if torch.cuda.is_available(): - torch.cuda.manual_seed_all(test_config["input_seed"]) + _seed_device(test_config["input_seed"], device) pos_x = {"x": torch.randn(test_config["batch_size"], test_config["dim"], dtype=dtype, device=device)} negative_kwargs = None if do_true_cfg: - torch.manual_seed(test_config["input_seed"] + 1) - if torch.cuda.is_available(): - torch.cuda.manual_seed_all(test_config["input_seed"] + 1) + _seed_device(test_config["input_seed"] + 1, device) neg_x = torch.randn(test_config["batch_size"], test_config["dim"], dtype=dtype, device=device) negative_kwargs = {"x": neg_x} @@ -425,19 +486,23 @@ def make_pipeline_and_inputs( # --------------------------------------------------------------------------- -# 3. isend_tensor_dict / irecv_tensor_dict (2 GPUs) +# 3. isend_tensor_dict / irecv_tensor_dict # --------------------------------------------------------------------------- -def isend_irecv_worker(local_rank: int, world_size: int, master_port: str, result_queue): - device = init_dist(local_rank, world_size, master_port) - initialize_model_parallel(pipeline_parallel_size=world_size) +def isend_irecv_worker( + local_rank: int, + world_size: int, + master_port: str, + device_kind: DeviceKind, + result_queue, +): + device = init_dist(local_rank, world_size, master_port, device_kind) + initialize_model_parallel(pipeline_parallel_size=world_size, backend=_mp_backend(device_kind)) pp_group = get_pp_group() if pp_group.is_first_rank: - torch.manual_seed(77) - if torch.cuda.is_available(): - torch.cuda.manual_seed_all(77) + _seed_device(77, device) tensor = torch.randn(3, 5, dtype=torch.float32, device=device) handles = pp_group.isend_tensor_dict({"t": tensor}) for h in handles: @@ -449,27 +514,42 @@ def isend_irecv_worker(local_rank: int, world_size: int, master_port: str, resul if torch.distributed.is_initialized(): torch.distributed.barrier() - destroy_distributed_env() + _cleanup_distributed() -@pytest.mark.gpu -@pytest.mark.skipif(current_omni_platform.get_device_count() < 2, reason="Need at least 2 GPUs") -@pytest.mark.parametrize("pp_size", [2]) -def test_isend_irecv_tensor_dict(pp_size: int): - """isend_tensor_dict / irecv_tensor_dict transfer a tensor dict without loss.""" +def _run_isend_irecv(pp_size: int, device_kind: DeviceKind, master_port: str) -> None: mp_context = torch.multiprocessing.get_context("spawn") manager = mp_context.Manager() q = manager.Queue() - - port = _find_free_port() - torch.multiprocessing.spawn(isend_irecv_worker, args=(pp_size, port, q), nprocs=pp_size) - + torch.multiprocessing.spawn( + isend_irecv_worker, + args=(pp_size, master_port, device_kind, q), + nprocs=pp_size, + ) results = {label: tensor for label, tensor in [q.get(), q.get()]} torch.testing.assert_close( results["received"], results["sent"], rtol=0, atol=0, msg="isend/irecv transferred tensor incorrectly" ) +@pytest.mark.core_model +@pytest.mark.diffusion +@pytest.mark.cpu +@pytest.mark.parametrize("pp_size", [2]) +def test_isend_irecv_tensor_dict(pp_size: int): + """isend_tensor_dict / irecv_tensor_dict transfer a tensor dict without loss.""" + _run_isend_irecv(pp_size, device_kind="cpu", master_port=_find_free_port()) + + +@pytest.mark.full_model +@pytest.mark.diffusion +@pytest.mark.parallel +@pytest.mark.parametrize("pp_size", [pytest.param(2, marks=_L4_TWO_GPU)]) +def test_isend_irecv_tensor_dict_parity(pp_size: int): + """Nightly: isend/irecv on real multi-GPU NCCL.""" + _run_isend_irecv(pp_size, device_kind="cuda", master_port=_find_free_port()) + + # --------------------------------------------------------------------------- # 4. predict_noise_maybe_with_cfg # --------------------------------------------------------------------------- @@ -477,12 +557,13 @@ def test_isend_irecv_tensor_dict(pp_size: int): _baseline_cache: dict[tuple, torch.Tensor] = {} -def compute_single_gpu_baseline(test_config: dict, dtype: torch.dtype, do_true_cfg: bool) -> torch.Tensor: - """Compute expected single-GPU output using the same MockPipelineParallel. - - Initializes a trivial distributed env (world_size=1) so that ``make_layers`` and the PP/CFG mixins work normally. - Results are cached so identical configs are only computed once. - """ +def compute_single_gpu_baseline( + test_config: dict, + dtype: torch.dtype, + do_true_cfg: bool, + device_kind: DeviceKind, +) -> torch.Tensor: + """Compute expected single-process output using the same MockPipelineParallel.""" key = ( test_config["num_layers"], test_config["dim"], @@ -492,27 +573,29 @@ def compute_single_gpu_baseline(test_config: dict, dtype: torch.dtype, do_true_c test_config["cfg_scale"], dtype, do_true_cfg, + device_kind, ) if key in _baseline_cache: return _baseline_cache[key] - device = init_dist(0, 1, _find_free_port()) - initialize_model_parallel(pipeline_parallel_size=1) + device = init_dist(0, 1, _find_free_port(), device_kind) + try: + initialize_model_parallel(pipeline_parallel_size=1, backend=_mp_backend(device_kind)) - pipeline, positive_kwargs, negative_kwargs = make_pipeline_and_inputs( - test_config, dtype, device, do_true_cfg=do_true_cfg - ) - - with torch.inference_mode(): - noise_pred = pipeline.predict_noise_maybe_with_cfg( - do_true_cfg=do_true_cfg, - true_cfg_scale=test_config["cfg_scale"], - positive_kwargs=positive_kwargs, - negative_kwargs=negative_kwargs, - cfg_normalize=False, + pipeline, positive_kwargs, negative_kwargs = make_pipeline_and_inputs( + test_config, dtype, device, do_true_cfg=do_true_cfg ) - destroy_distributed_env() + with torch.inference_mode(): + noise_pred = pipeline.predict_noise_maybe_with_cfg( + do_true_cfg=do_true_cfg, + true_cfg_scale=test_config["cfg_scale"], + positive_kwargs=positive_kwargs, + negative_kwargs=negative_kwargs, + cfg_normalize=False, + ) + finally: + _cleanup_distributed() _baseline_cache[key] = noise_pred.cpu() return _baseline_cache[key] @@ -527,11 +610,16 @@ def predict_noise_worker( do_true_cfg: bool, dtype: torch.dtype, test_config: dict, + device_kind: DeviceKind, result_queue, ): """Generic predict-noise worker parameterized by PP and CFG topology.""" - device = init_dist(local_rank, world_size, master_port) - initialize_model_parallel(pipeline_parallel_size=pp_size, cfg_parallel_size=cfg_size) + device = init_dist(local_rank, world_size, master_port, device_kind) + initialize_model_parallel( + pipeline_parallel_size=pp_size, + cfg_parallel_size=cfg_size, + backend=_mp_backend(device_kind), + ) pp_group = get_pp_group() cfg_rank = get_classifier_free_guidance_rank() @@ -548,8 +636,6 @@ def predict_noise_worker( negative_kwargs=negative_kwargs, cfg_normalize=False, ) - # This worker exercises predict_noise_maybe_with_cfg directly, bypassing diffuse(). - # Flush the non-last PP rank's async send before barrier / process teardown. pipeline._sync_pp_send() if pp_group.is_last_rank: @@ -561,77 +647,22 @@ def predict_noise_worker( if torch.distributed.is_initialized(): torch.distributed.barrier() - destroy_distributed_env() + _cleanup_distributed() -@pytest.mark.gpu -@pytest.mark.parametrize( - "pp_size, cfg_size, do_true_cfg, dtype, num_layers, input_seed, rtol, atol", - [ - pytest.param( - 2, - 1, - False, - torch.float32, - 4, - 100, - 1e-5, - 1e-5, - marks=pytest.mark.skipif(current_omni_platform.get_device_count() < 2, reason="Need at least 2 GPUs"), - id="pp2-no_cfg-float32", - ), - pytest.param( - 2, - 1, - False, - torch.bfloat16, - 4, - 100, - 1e-2, - 1e-2, - marks=pytest.mark.skipif(current_omni_platform.get_device_count() < 2, reason="Need at least 2 GPUs"), - id="pp2-no_cfg-bfloat16", - ), - pytest.param( - 2, - 1, - True, - torch.bfloat16, - 4, - 100, - 1e-2, - 1e-2, - marks=pytest.mark.skipif(current_omni_platform.get_device_count() < 2, reason="Need at least 2 GPUs"), - id="pp2-seq_cfg-bfloat16", - ), - pytest.param( - 2, - 2, - True, - torch.bfloat16, - 4, - 100, - 1e-2, - 1e-2, - marks=pytest.mark.skipif(current_omni_platform.get_device_count() < 4, reason="Need at least 4 GPUs"), - id="pp2-cfg2-bfloat16", - ), - pytest.param( - 3, - 1, - False, - torch.bfloat16, - 6, - 100, - 1e-2, - 1e-2, - marks=pytest.mark.skipif(current_omni_platform.get_device_count() < 3, reason="Need at least 3 GPUs"), - id="pp3-no_cfg-bfloat16", - ), - ], -) -def test_predict_noise(pp_size, cfg_size, do_true_cfg, dtype, num_layers, input_seed, rtol, atol): - """predict_noise_maybe_with_cfg output matches the single-GPU baseline across PP / CFG topologies.""" +def _run_predict_noise( + *, + pp_size: int, + cfg_size: int, + do_true_cfg: bool, + dtype: torch.dtype, + num_layers: int, + input_seed: int, + rtol: float, + atol: float, + device_kind: DeviceKind, + master_port: str, +) -> None: test_config = { "num_layers": num_layers, "dim": 64, @@ -641,22 +672,20 @@ def test_predict_noise(pp_size, cfg_size, do_true_cfg, dtype, num_layers, input_ "input_seed": input_seed, } - baseline_out = compute_single_gpu_baseline(test_config, dtype, do_true_cfg) + baseline_out = compute_single_gpu_baseline(test_config, dtype, do_true_cfg, device_kind) mp_context = torch.multiprocessing.get_context("spawn") manager = mp_context.Manager() pp_q = manager.Queue() world_size = pp_size * cfg_size - port = _find_free_port() torch.multiprocessing.spawn( predict_noise_worker, - args=(world_size, port, pp_size, cfg_size, do_true_cfg, dtype, test_config, pp_q), + args=(world_size, master_port, pp_size, cfg_size, do_true_cfg, dtype, test_config, device_kind, pp_q), nprocs=world_size, ) pp_out = pp_q.get() - assert baseline_out.shape == pp_out.shape torch.testing.assert_close( pp_out, @@ -667,35 +696,157 @@ def test_predict_noise(pp_size, cfg_size, do_true_cfg, dtype, num_layers, input_ ) +_CPU_PREDICT_NOISE_CASES = [ + pytest.param(2, 1, False, 4, 100, 1e-5, 1e-5, id="pp2-no_cfg"), + pytest.param(2, 1, True, 4, 100, 1e-5, 1e-5, id="pp2-seq_cfg"), + pytest.param(2, 2, True, 4, 100, 1e-5, 1e-5, id="pp2-cfg2"), + pytest.param(3, 1, False, 6, 100, 1e-5, 1e-5, id="pp3-no_cfg"), +] + +_GPU_PREDICT_NOISE_CASES = [ + pytest.param( + 2, + 1, + False, + torch.float32, + 4, + 100, + 1e-5, + 1e-5, + marks=_L4_TWO_GPU, + id="pp2-no_cfg-float32", + ), + pytest.param( + 2, + 1, + False, + torch.bfloat16, + 4, + 100, + 1e-2, + 1e-2, + marks=_L4_TWO_GPU, + id="pp2-no_cfg-bfloat16", + ), + pytest.param( + 2, + 1, + True, + torch.bfloat16, + 4, + 100, + 1e-2, + 1e-2, + marks=_L4_TWO_GPU, + id="pp2-seq_cfg-bfloat16", + ), + pytest.param( + 2, + 2, + True, + torch.bfloat16, + 4, + 100, + 1e-2, + 1e-2, + marks=_L4_FOUR_GPU, + id="pp2-cfg2-bfloat16", + ), + pytest.param( + 3, + 1, + False, + torch.bfloat16, + 6, + 100, + 1e-2, + 1e-2, + marks=_L4_THREE_GPU, + id="pp3-no_cfg-bfloat16", + ), +] + + +@pytest.mark.core_model +@pytest.mark.diffusion +@pytest.mark.cpu +@pytest.mark.parametrize( + "pp_size, cfg_size, do_true_cfg, num_layers, input_seed, rtol, atol", + _CPU_PREDICT_NOISE_CASES, +) +def test_predict_noise(pp_size, cfg_size, do_true_cfg, num_layers, input_seed, rtol, atol): + """predict_noise_maybe_with_cfg output matches baseline across PP / CFG topologies.""" + _run_predict_noise( + pp_size=pp_size, + cfg_size=cfg_size, + do_true_cfg=do_true_cfg, + dtype=torch.float32, + num_layers=num_layers, + input_seed=input_seed, + rtol=rtol, + atol=atol, + device_kind="cpu", + master_port=_find_free_port(), + ) + + +@pytest.mark.full_model +@pytest.mark.diffusion +@pytest.mark.parallel +@pytest.mark.parametrize( + "pp_size, cfg_size, do_true_cfg, dtype, num_layers, input_seed, rtol, atol", + _GPU_PREDICT_NOISE_CASES, +) +def test_predict_noise_parity(pp_size, cfg_size, do_true_cfg, dtype, num_layers, input_seed, rtol, atol): + """Nightly: predict_noise parity on real multi-GPU NCCL.""" + _run_predict_noise( + pp_size=pp_size, + cfg_size=cfg_size, + do_true_cfg=do_true_cfg, + dtype=dtype, + num_layers=num_layers, + input_seed=input_seed, + rtol=rtol, + atol=atol, + device_kind="cuda", + master_port=_find_free_port(), + ) + + # --------------------------------------------------------------------------- # 5. scheduler_step_maybe_with_cfg # --------------------------------------------------------------------------- -def compute_scheduler_step_baseline(test_config: dict, do_true_cfg: bool) -> torch.Tensor: - """Single-GPU reference: predict_noise + scheduler_step.""" - device = init_dist(0, 1, _find_free_port()) - initialize_model_parallel(pipeline_parallel_size=1) +def compute_scheduler_step_baseline( + test_config: dict, + do_true_cfg: bool, + device_kind: DeviceKind, +) -> torch.Tensor: + """Single-process reference: predict_noise + scheduler_step.""" + device = init_dist(0, 1, _find_free_port(), device_kind) + try: + initialize_model_parallel(pipeline_parallel_size=1, backend=_mp_backend(device_kind)) - pipeline, positive_kwargs, negative_kwargs = make_pipeline_and_inputs( - test_config, torch.float32, device, do_true_cfg=do_true_cfg - ) - latents = positive_kwargs["x"] - t = torch.tensor(500, device=device) - - with torch.inference_mode(): - noise_pred = pipeline.predict_noise_maybe_with_cfg( - do_true_cfg=do_true_cfg, - true_cfg_scale=test_config["cfg_scale"], - positive_kwargs=positive_kwargs, - negative_kwargs=negative_kwargs, - cfg_normalize=False, - ) - result = pipeline.scheduler_step_maybe_with_cfg( - noise_pred=noise_pred, t=t, latents=latents, do_true_cfg=do_true_cfg + pipeline, positive_kwargs, negative_kwargs = make_pipeline_and_inputs( + test_config, torch.float32, device, do_true_cfg=do_true_cfg ) + latents = positive_kwargs["x"] + t = torch.tensor(500, device=device) - destroy_distributed_env() + with torch.inference_mode(): + noise_pred = pipeline.predict_noise_maybe_with_cfg( + do_true_cfg=do_true_cfg, + true_cfg_scale=test_config["cfg_scale"], + positive_kwargs=positive_kwargs, + negative_kwargs=negative_kwargs, + cfg_normalize=False, + ) + result = pipeline.scheduler_step_maybe_with_cfg( + noise_pred=noise_pred, t=t, latents=latents, do_true_cfg=do_true_cfg + ) + finally: + _cleanup_distributed() return result.cpu() @@ -707,10 +858,15 @@ def scheduler_step_worker( cfg_size: int, do_true_cfg: bool, test_config: dict, + device_kind: DeviceKind, result_queue, ): - device = init_dist(local_rank, world_size, master_port) - initialize_model_parallel(pipeline_parallel_size=pp_size, cfg_parallel_size=cfg_size) + device = init_dist(local_rank, world_size, master_port, device_kind) + initialize_model_parallel( + pipeline_parallel_size=pp_size, + cfg_parallel_size=cfg_size, + backend=_mp_backend(device_kind), + ) pp_group = get_pp_group() cfg_rank = get_classifier_free_guidance_rank() @@ -732,43 +888,25 @@ def scheduler_step_worker( latents = pipeline.scheduler_step_maybe_with_cfg( noise_pred=noise_pred, t=t, latents=latents, do_true_cfg=do_true_cfg ) - # This worker exercises scheduler_step_maybe_with_cfg directly, bypassing diffuse(). - # Flush the last PP rank's async latent send before barrier / process teardown. pipeline._sync_pp_send() if pp_group.is_first_rank and cfg_rank == 0: - resolved = latents.contiguous() - result_queue.put(resolved.cpu()) + result_queue.put(latents.contiguous().cpu()) if torch.distributed.is_initialized(): torch.distributed.barrier() - destroy_distributed_env() + _cleanup_distributed() -@pytest.mark.gpu -@pytest.mark.parametrize( - "pp_size, cfg_size, do_true_cfg, input_seed", - [ - pytest.param( - 2, - 1, - False, - 300, - marks=pytest.mark.skipif(current_omni_platform.get_device_count() < 2, reason="Need at least 2 GPUs"), - id="pp2-no_cfg", - ), - pytest.param( - 2, - 2, - True, - 600, - marks=pytest.mark.skipif(current_omni_platform.get_device_count() < 4, reason="Need at least 4 GPUs"), - id="pp2-cfg2-true_cfg", - ), - ], -) -def test_scheduler_step(pp_size, cfg_size, do_true_cfg, input_seed): - """Rank 0 latents after scheduler_step match the single-GPU baseline across PP / CFG topologies.""" +def _run_scheduler_step( + *, + pp_size: int, + cfg_size: int, + do_true_cfg: bool, + input_seed: int, + device_kind: DeviceKind, + master_port: str, +) -> None: test_config = { "num_layers": 4, "dim": 64, @@ -778,17 +916,16 @@ def test_scheduler_step(pp_size, cfg_size, do_true_cfg, input_seed): "input_seed": input_seed, } - baseline = compute_scheduler_step_baseline(test_config, do_true_cfg) + baseline = compute_scheduler_step_baseline(test_config, do_true_cfg, device_kind) mp_context = torch.multiprocessing.get_context("spawn") manager = mp_context.Manager() q = manager.Queue() - port = _find_free_port() world_size = pp_size * cfg_size torch.multiprocessing.spawn( scheduler_step_worker, - args=(world_size, port, pp_size, cfg_size, do_true_cfg, test_config, q), + args=(world_size, master_port, pp_size, cfg_size, do_true_cfg, test_config, device_kind, q), nprocs=world_size, ) @@ -800,3 +937,47 @@ def test_scheduler_step(pp_size, cfg_size, do_true_cfg, input_seed): atol=0, msg=f"PP={pp_size} CFG={cfg_size} scheduler step latents on rank 0 do not match single-GPU baseline", ) + + +@pytest.mark.core_model +@pytest.mark.diffusion +@pytest.mark.cpu +@pytest.mark.parametrize( + "pp_size, cfg_size, do_true_cfg, input_seed", + [ + pytest.param(2, 1, False, 300, id="pp2-no_cfg"), + pytest.param(2, 2, True, 600, id="pp2-cfg2-true_cfg"), + ], +) +def test_scheduler_step(pp_size, cfg_size, do_true_cfg, input_seed): + """Rank 0 latents after scheduler_step match baseline across PP / CFG topologies.""" + _run_scheduler_step( + pp_size=pp_size, + cfg_size=cfg_size, + do_true_cfg=do_true_cfg, + input_seed=input_seed, + device_kind="cpu", + master_port=_find_free_port(), + ) + + +@pytest.mark.full_model +@pytest.mark.diffusion +@pytest.mark.parallel +@pytest.mark.parametrize( + "pp_size, cfg_size, do_true_cfg, input_seed", + [ + pytest.param(2, 1, False, 300, marks=_L4_TWO_GPU, id="pp2-no_cfg"), + pytest.param(2, 2, True, 600, marks=_L4_FOUR_GPU, id="pp2-cfg2-true_cfg"), + ], +) +def test_scheduler_step_parity(pp_size, cfg_size, do_true_cfg, input_seed): + """Nightly: scheduler_step parity on real multi-GPU NCCL.""" + _run_scheduler_step( + pp_size=pp_size, + cfg_size=cfg_size, + do_true_cfg=do_true_cfg, + input_seed=input_seed, + device_kind="cuda", + master_port=_find_free_port(), + ) diff --git a/tests/diffusion/distributed/test_zimage_parallelism.py b/tests/diffusion/distributed/test_tensor_parallel.py similarity index 53% rename from tests/diffusion/distributed/test_zimage_parallelism.py rename to tests/diffusion/distributed/test_tensor_parallel.py index ab330ee9a..9f39112ae 100644 --- a/tests/diffusion/distributed/test_zimage_parallelism.py +++ b/tests/diffusion/distributed/test_tensor_parallel.py @@ -1,16 +1,14 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Z-Image end-to-end tests for diffusion parallelism. +"""DiT tensor-parallel parity and perf checks for diffusion models. -This file currently covers: -- DiT tensor parallelism (TP=2) vs TP=1. -- VAE patch parallelism (vae_patch_parallel_size=2) vs baseline on TP=2. - -Note: CUDA-only (>=2 GPUs). We use `enforce_eager=False` (default) to enable -`torch.compile`. +Compares TP=2 against TP=1 on output correctness, latency, and peak memory. +Uses ``Tongyi-MAI/Z-Image-Turbo`` as the parity model. """ +from __future__ import annotations + import os import time from pathlib import Path @@ -31,22 +29,15 @@ from vllm_omni.platforms import current_omni_platform os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" PROMPT = "a photo of a cat sitting on a laptop keyboard" - - -def _get_zimage_model() -> str: - # Allow overriding the model for local/offline environments. - # Can be either a HuggingFace repo id or a local path. - return os.environ.get("VLLM_TEST_ZIMAGE_MODEL", "Tongyi-MAI/Z-Image-Turbo") +DEFAULT_MODEL = "Tongyi-MAI/Z-Image-Turbo" def _pil_to_float_rgb_tensor(img: Image.Image) -> torch.Tensor: - """Convert PIL image to float32 RGB tensor in [0, 1] with shape [H, W, 3].""" arr = np.asarray(img.convert("RGB"), dtype=np.float32) / 255.0 return torch.from_numpy(arr) def _diff_metrics(a: Image.Image, b: Image.Image) -> tuple[float, float]: - """Return (mean_abs_diff, p99_abs_diff) over RGB pixels in [0, 1].""" ta = _pil_to_float_rgb_tensor(a) tb = _pil_to_float_rgb_tensor(b) assert ta.shape == tb.shape, f"Image shapes differ: {ta.shape} vs {tb.shape}" @@ -71,7 +62,7 @@ def _extract_single_image(outputs) -> Image.Image: return images[0] -def _run_zimage_generate( +def _run_generate( *, tp_size: int, height: int, @@ -79,8 +70,6 @@ def _run_zimage_generate( num_inference_steps: int, seed: int, enforce_eager: bool, - vae_use_tiling: bool = False, - vae_patch_parallel_size: int = 1, num_requests: int = 4, ) -> tuple[Image.Image, float, float]: if num_requests < 2: @@ -91,25 +80,11 @@ def _run_zimage_generate( monitor = DeviceMemoryMonitor(device_index=device_index, interval=0.02) monitor.start() try: - # Each run needs a distinct DiffusionParallelConfig; use OmniRunner per call (not the - # parametrized omni_runner fixture, which is fixed per module). with OmniRunner( - _get_zimage_model(), - parallel_config=DiffusionParallelConfig( - tensor_parallel_size=tp_size, - vae_patch_parallel_size=vae_patch_parallel_size, - ), + DEFAULT_MODEL, + parallel_config=DiffusionParallelConfig(tensor_parallel_size=tp_size), enforce_eager=enforce_eager, - vae_use_tiling=vae_use_tiling, ) as runner: - # NOTE: Omni closes itself when a generate() call is exhausted. - # To avoid measuring teardown time (process shutdown, memory cleanup), - # we measure the latency to produce *subsequent* outputs within a single - # generator run. - # - # This also serves as a warmup: the first output may include extra - # compilation/caching overhead, while later outputs are closer to - # steady-state inference. gen = runner.omni.generate( [PROMPT] * num_requests, OmniDiffusionSamplingParams( @@ -134,14 +109,11 @@ def _run_zimage_generate( per_request_times_s.append(t_now - t_prev) t_prev = t_now - # Ensure the generator is fully consumed so it can clean up. for _ in gen: pass median_time_s = float(np.median(per_request_times_s)) - peak_memory_mb = monitor.peak_used_mb - return _extract_single_image([last_output]), median_time_s, peak_memory_mb finally: monitor.stop() @@ -151,20 +123,20 @@ def _run_zimage_generate( @pytest.mark.diffusion @pytest.mark.parallel @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) -def test_zimage_tensor_parallel_tp2(tmp_path: Path): +def test_tensor_parallel_tp2(tmp_path: Path): if current_omni_platform.is_npu(): - pytest.skip("Z-Image TP e2e test is only supported on CUDA and ROCm for now.") + pytest.skip("Tensor parallel parity test is only supported on CUDA and ROCm for now.") if not current_omni_platform.is_available() or current_omni_platform.device_count() < 2: - pytest.skip("Z-Image TP=2 requires >= 2 devices.") + pytest.skip("Tensor parallel TP=2 requires >= 2 devices.") + model_name = DEFAULT_MODEL enforce_eager = False - height = 512 width = 512 num_inference_steps = 2 seed = 42 - tp1_img, tp1_time_s, tp1_peak_mem = _run_zimage_generate( + tp1_img, tp1_time_s, tp1_peak_mem = _run_generate( tp_size=1, height=height, width=width, @@ -172,7 +144,7 @@ def test_zimage_tensor_parallel_tp2(tmp_path: Path): seed=seed, enforce_eager=enforce_eager, ) - tp2_img, tp2_time_s, tp2_peak_mem = _run_zimage_generate( + tp2_img, tp2_time_s, tp2_peak_mem = _run_generate( tp_size=2, height=height, width=width, @@ -181,8 +153,8 @@ def test_zimage_tensor_parallel_tp2(tmp_path: Path): enforce_eager=enforce_eager, ) - tp1_path = tmp_path / "zimage_tp1.png" - tp2_path = tmp_path / "zimage_tp2.png" + tp1_path = tmp_path / "tp1.png" + tp2_path = tmp_path / "tp2.png" tp1_img.save(tp1_path) tp2_img.save(tp2_path) @@ -193,7 +165,7 @@ def test_zimage_tensor_parallel_tp2(tmp_path: Path): mean_threshold = 3e-2 p99_threshold = 2.5e-1 print( - "Z-Image TP image diff stats (TP=1 vs TP=2): " + f"{model_name} TP image diff (TP=1 vs TP=2): " f"mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e}; " f"thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e}; " f"tp1_img={tp1_path}, tp2_img={tp2_path}" @@ -203,73 +175,11 @@ def test_zimage_tensor_parallel_tp2(tmp_path: Path): f"(thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e})" ) - print(f"Z-Image TP perf (lower is better): tp1_time_s={tp1_time_s:.6f}, tp2_time_s={tp2_time_s:.6f}") - # ROCm is not optimized TP2 can be slower than TP1 + print(f"TP perf (lower is better): tp1_time_s={tp1_time_s:.6f}, tp2_time_s={tp2_time_s:.6f}") if not current_omni_platform.is_rocm(): assert tp2_time_s < tp1_time_s, f"Expected TP=2 to be faster than TP=1 (tp1={tp1_time_s}, tp2={tp2_time_s})" - print(f"Z-Image TP peak memory (MB): tp1_peak_mem={tp1_peak_mem:.2f}, tp2_peak_mem={tp2_peak_mem:.2f}") + print(f"TP peak memory (MB): tp1_peak_mem={tp1_peak_mem:.2f}, tp2_peak_mem={tp2_peak_mem:.2f}") assert tp2_peak_mem < tp1_peak_mem, ( f"Expected TP=2 to use less peak memory than TP=1 (tp1={tp1_peak_mem}, tp2={tp2_peak_mem})" ) - - -@pytest.mark.advanced_model -@pytest.mark.diffusion -@pytest.mark.parallel -@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) -def test_zimage_vae_patch_parallel_tp2(tmp_path: Path): - if current_omni_platform.is_npu(): - pytest.skip("Z-Image VAE patch parallel e2e test is only supported on CUDA and ROCm for now.") - if not current_omni_platform.is_available() or current_omni_platform.device_count() < 2: - pytest.skip("Z-Image VAE patch parallel TP=2 requires >= 2 devices.") - - enforce_eager = False - - # Use a larger image to ensure there are multiple VAE tiles. - height = 1152 - width = 1152 - num_inference_steps = 2 - seed = 42 - - baseline_img, _baseline_time_s, _baseline_peak_mem = _run_zimage_generate( - tp_size=2, - height=height, - width=width, - num_inference_steps=num_inference_steps, - seed=seed, - enforce_eager=enforce_eager, - vae_use_tiling=True, - vae_patch_parallel_size=1, - num_requests=2, - ) - pp2_img, _pp2_time_s, _pp2_peak_mem = _run_zimage_generate( - tp_size=2, - height=height, - width=width, - num_inference_steps=num_inference_steps, - seed=seed, - enforce_eager=enforce_eager, - vae_use_tiling=True, - vae_patch_parallel_size=2, - num_requests=2, - ) - - baseline_path = tmp_path / "zimage_tp2_vae_pp1.png" - pp2_path = tmp_path / "zimage_tp2_vae_pp2.png" - baseline_img.save(baseline_path) - pp2_img.save(pp2_path) - - mean_abs_diff, p99_abs_diff = _diff_metrics(baseline_img, pp2_img) - mean_threshold = 5e-3 - p99_threshold = 1e-1 - print( - "Z-Image VAE patch parallel image diff stats (TP=2, pp=1 vs pp=2): " - f"mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e}; " - f"thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e}; " - f"pp1_img={baseline_path}, pp2_img={pp2_path}" - ) - assert mean_abs_diff <= mean_threshold and p99_abs_diff <= p99_threshold, ( - f"Image diff exceeded threshold: mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e} " - f"(thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e})" - ) diff --git a/tests/diffusion/distributed/test_vae_decode_parallelism.py b/tests/diffusion/distributed/test_vae_decode_parallelism.py index 2fd531e3e..57fe1e073 100644 --- a/tests/diffusion/distributed/test_vae_decode_parallelism.py +++ b/tests/diffusion/distributed/test_vae_decode_parallelism.py @@ -1,151 +1,181 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""VAE patch-parallel decode parity tests for diffusion models. + +Compares ``vae_patch_parallel_size=2`` against ``vae_patch_parallel_size=1`` on +TP=2 with VAE tiling enabled. Covers one video model (Wan T2V) and one image +model (Qwen-Image). CUDA/ROCm-only (>=2 devices). +""" + +from __future__ import annotations + import os -import sys from pathlib import Path +from typing import Any import numpy as np import pytest import torch from PIL import Image -from vllm.distributed.parallel_state import cleanup_dist_env_and_memory +from tests.helpers.mark import hardware_test +from tests.helpers.runtime import OmniRunner from vllm_omni.diffusion.data import DiffusionParallelConfig from vllm_omni.inputs.data import OmniDiffusionSamplingParams - -# ruff: noqa: E402 -REPO_ROOT = Path(__file__).resolve().parents[2] -if str(REPO_ROOT) not in sys.path: - sys.path.insert(0, str(REPO_ROOT)) - -import time - -from tests.helpers.runtime import OmniRunner from vllm_omni.platforms import current_omni_platform os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -all_model_configs = [ - { - "model_name": "Wan-AI/Wan2.2-TI2V-5B-Diffusers", - "out_width": 1280, - "out_height": 720, - "out_frames": 16, - }, - { - "model_name": "Wan-AI/Wan2.2-T2V-A14B-Diffusers", - "out_width": 1280, - "out_height": 720, - "out_frames": 16, - }, - { - "model_name": "Wan-AI/Wan2.2-I2V-A14B-Diffusers", - "out_width": 1280, - "out_height": 720, - "out_frames": 16, - }, - { - "model_name": "Qwen/Qwen-Image", - "out_width": 3072, - "out_height": 3072, - "out_frames": 1, - }, - # { - # "model_name": "stepfun-ai/NextStep-1.1", - # "out_width": 3072, - # "out_height": 3072, - # "out_frames": 1, - # }, +PROMPT = "A cat sitting on a table" + +MODEL_CASES = [ + pytest.param( + { + "model_name": "Wan-AI/Wan2.2-T2V-A14B-Diffusers", + "height": 512, + "width": 512, + "num_frames": 5, + "needs_image": False, + "is_moe": False, + "mean_threshold": 3e-2, + "p99_threshold": 3e-2, + }, + id="wan22_t2v_a14b", + ), + pytest.param( + { + "model_name": "Qwen/Qwen-Image", + "height": 1152, + "width": 1152, + "num_frames": 1, + "needs_image": False, + "is_moe": False, + "mean_threshold": 5e-3, + "p99_threshold": 1e-1, + }, + id="qwen_image", + ), ] -def is_nextstep_model(model_name: str) -> bool: - """Check if the model is a NextStep model by reading its config.""" - from vllm.transformers_utils.config import get_hf_file_to_dict - - try: - cfg = get_hf_file_to_dict("config.json", model_name) - if cfg and cfg.get("model_type") == "nextstep": - return True - except Exception: - pass - return False +def _to_float_array(data: Any) -> np.ndarray: + if isinstance(data, Image.Image): + return np.asarray(data.convert("RGB"), dtype=np.float32) / 255.0 + if isinstance(data, torch.Tensor): + arr = data.detach().cpu().numpy() + else: + arr = np.asarray(data) + if arr.dtype == np.uint8: + return arr.astype(np.float32) / 255.0 + return arr.astype(np.float32) -def model_run(model_configs, tp, out_height, out_width, out_frames, using_tile, vae_patch_parallel_size=1): +def _extract_output_array(output: Any) -> np.ndarray: + req_out = output.request_output + payload = req_out.images[0] + return _to_float_array(payload) + + +def _diff_metrics(a: np.ndarray, b: np.ndarray) -> tuple[float, float]: + assert a.shape == b.shape, f"Output shapes differ: {a.shape} vs {b.shape}" + abs_diff = np.abs(a - b) + return float(abs_diff.mean()), float(np.quantile(abs_diff, 0.99)) + + +def _save_preview(arr: np.ndarray, path: Path) -> None: + """Save a single RGB frame preview; video tensors use the first frame.""" + preview = np.asarray(arr) + # (B, T, H, W, C) video + if preview.ndim == 5: + preview = preview[0, 0] + # (T, H, W, C) video or (B, H, W, C) image batch + elif preview.ndim == 4: + preview = preview[0] + if preview.ndim != 3 or preview.shape[-1] not in (3, 4): + raise ValueError(f"Expected HWC RGB(A) preview, got shape {arr.shape} -> {preview.shape}") + preview = np.clip(preview[..., :3], 0.0, 1.0) + Image.fromarray((preview * 255.0).astype(np.uint8)).save(path) + + +def _build_sampling_params(model_case: dict[str, Any], seed: int) -> OmniDiffusionSamplingParams: + kwargs: dict[str, Any] = { + "height": model_case["height"], + "width": model_case["width"], + "num_frames": model_case["num_frames"], + "num_inference_steps": 2, + "seed": seed, + } + if model_case["model_name"].startswith("Wan-AI/"): + kwargs["guidance_scale"] = 4.0 + if model_case["is_moe"]: + kwargs["guidance_scale_2"] = 1.0 + kwargs["boundary_ratio"] = 0.5 + return OmniDiffusionSamplingParams(**kwargs) + + +def _run_generate( + *, + model_case: dict[str, Any], + vae_patch_parallel_size: int, + seed: int, +) -> np.ndarray: + current_omni_platform.empty_cache() parallel_config = DiffusionParallelConfig( - tensor_parallel_size=tp, + tensor_parallel_size=2, vae_patch_parallel_size=vae_patch_parallel_size, ) - - omni_kwargs = { - "vae_use_tiling": using_tile, - "parallel_config": parallel_config, - } - use_nextstep = is_nextstep_model(model_configs["model_name"]) - if use_nextstep: - # NextStep-1.1 requires explicit pipeline class - omni_kwargs["model_class_name"] = "NextStep11Pipeline" - - with OmniRunner(model_configs["model_name"], **omni_kwargs) as runner: - m = runner.omni - image = Image.new("RGB", (out_width, out_height), (0, 0, 0)) - start = time.perf_counter() - outputs = m.generate( - { - "prompt": "A cat sitting on a table", - "multi_modal_data": {"image": image}, - }, - sampling_params_list=OmniDiffusionSamplingParams( - height=model_configs["out_height"], - width=model_configs["out_width"], - num_frames=model_configs["out_frames"], - num_inference_steps=2, - generator=torch.Generator(current_omni_platform.device_type).manual_seed(42), - ), + with OmniRunner( + model_case["model_name"], + parallel_config=parallel_config, + vae_use_tiling=True, + enforce_eager=False, + ) as runner: + request: dict[str, Any] = {"prompt": model_case.get("prompt", PROMPT)} + if model_case["needs_image"]: + request["multi_modal_data"] = { + "image": Image.new("RGB", (model_case["width"], model_case["height"]), (0, 0, 0)) + } + outputs = runner.omni.generate( + request, + sampling_params_list=_build_sampling_params(model_case, seed), ) - end = time.perf_counter() - first_output = outputs[0] - req_out = first_output.request_output - frames = req_out.images[0] - if isinstance(frames, torch.Tensor): - frames = frames.detach().cpu().numpy() - elif isinstance(frames, Image.Image): - frames = np.array(frames) - # frames shape: (batch, num_frames, height, width, channels) - cost = (end - start) * 1000 - return frames, cost - cleanup_dist_env_and_memory() + return _extract_output_array(outputs[0]) -@pytest.mark.parametrize("model_configs", all_model_configs) -def test_vae_parallel_model(model_configs: str): - parallel_result, parallel_time = model_run( - model_configs=model_configs, - tp=2, - out_width=1280, - out_height=704, - out_frames=5, - using_tile=True, - vae_patch_parallel_size=2, - ) - non_parallel_result, non_parallel_time = model_run( - model_configs=model_configs, - tp=2, - out_width=1280, - out_height=704, - out_frames=5, - using_tile=True, - vae_patch_parallel_size=1, - ) - result_diff = np.abs(non_parallel_result - parallel_result) +@pytest.mark.full_model +@pytest.mark.diffusion +@pytest.mark.parallel +@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) +@pytest.mark.parametrize("model_case", MODEL_CASES) +def test_vae_patch_parallel_tp2(model_case: dict[str, Any], tmp_path: Path): + if current_omni_platform.is_npu(): + pytest.skip("VAE patch parallel test is only supported on CUDA and ROCm for now.") + if not current_omni_platform.is_available() or current_omni_platform.device_count() < 2: + pytest.skip("VAE patch parallel TP=2 requires >= 2 devices.") - mean_threshold = 3e-2 - max_threshold = 3e-2 # they should be totally same + seed = 42 + model_id = model_case["model_name"].split("/")[-1] + + baseline = _run_generate(model_case=model_case, vae_patch_parallel_size=1, seed=seed) + parallel = _run_generate(model_case=model_case, vae_patch_parallel_size=2, seed=seed) + + baseline_path = tmp_path / f"{model_id}_tp2_vae_pp1.png" + parallel_path = tmp_path / f"{model_id}_tp2_vae_pp2.png" + _save_preview(baseline, baseline_path) + _save_preview(parallel, parallel_path) + + mean_abs_diff, p99_abs_diff = _diff_metrics(baseline, parallel) + mean_threshold = model_case["mean_threshold"] + p99_threshold = model_case["p99_threshold"] print( - f"{model_configs['model_name']} TP = 2 (tile + parallel vs tile): " - f"mean_abs_diff={result_diff.mean():.6e}, max_abs_diff={result_diff.max():.6e}; " - f"thresholds: mean<={mean_threshold:.6e}, max<={max_threshold:.6e}; " - f"parallel generate take time: {parallel_time:.2f} ms, non-parallel take time: {non_parallel_time:.2f} ms" + f"{model_case['model_name']} VAE patch parallel diff (TP=2, pp=1 vs pp=2): " + f"mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e}; " + f"thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e}; " + f"pp1_preview={baseline_path}, pp2_preview={parallel_path}" + ) + assert mean_abs_diff <= mean_threshold and p99_abs_diff <= p99_threshold, ( + f"Output diff exceeded threshold: mean_abs_diff={mean_abs_diff:.6e}, " + f"p99_abs_diff={p99_abs_diff:.6e} " + f"(thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e})" ) - # assert non_parallel_time > parallel_time - assert result_diff.mean() < mean_threshold - assert result_diff.max() < max_threshold diff --git a/tests/diffusion/hooks/test_hook_registry.py b/tests/diffusion/hooks/test_hook_registry.py index 6c8535cfe..3ee93a073 100644 --- a/tests/diffusion/hooks/test_hook_registry.py +++ b/tests/diffusion/hooks/test_hook_registry.py @@ -12,6 +12,9 @@ from torch import nn from vllm_omni.diffusion.hooks.base import HookRegistry, ModelHook +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + + DEFAULT_OUT = "ECHO" OVERRIDE_OUT = "OVERRIDE" INPUT_KWARG = "inp" diff --git a/tests/diffusion/layers/test_rotary_emb_equivalence.py b/tests/diffusion/layers/test_rotary_emb_equivalence.py index 2fbb7a31f..09aef0373 100644 --- a/tests/diffusion/layers/test_rotary_emb_equivalence.py +++ b/tests/diffusion/layers/test_rotary_emb_equivalence.py @@ -13,6 +13,8 @@ from __future__ import annotations import pytest import torch +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + def _apply_rotary_emb_helios_original( hidden_states: torch.Tensor, diff --git a/tests/e2e/offline_inference/rlhf_test/__init__.py b/tests/diffusion/models/dreamzero/__init__.py similarity index 100% rename from tests/e2e/offline_inference/rlhf_test/__init__.py rename to tests/diffusion/models/dreamzero/__init__.py diff --git a/tests/dreamzero/test_crossattn_cache.py b/tests/diffusion/models/dreamzero/test_crossattn_cache.py similarity index 100% rename from tests/dreamzero/test_crossattn_cache.py rename to tests/diffusion/models/dreamzero/test_crossattn_cache.py diff --git a/tests/dreamzero/test_deploy_configs.py b/tests/diffusion/models/dreamzero/test_deploy_configs.py similarity index 100% rename from tests/dreamzero/test_deploy_configs.py rename to tests/diffusion/models/dreamzero/test_deploy_configs.py diff --git a/tests/dreamzero/test_dreamzero_utils.py b/tests/diffusion/models/dreamzero/test_dreamzero_utils.py similarity index 100% rename from tests/dreamzero/test_dreamzero_utils.py rename to tests/diffusion/models/dreamzero/test_dreamzero_utils.py diff --git a/tests/dreamzero/test_fused_qk_rms_norm.py b/tests/diffusion/models/dreamzero/test_fused_qk_rms_norm.py similarity index 100% rename from tests/dreamzero/test_fused_qk_rms_norm.py rename to tests/diffusion/models/dreamzero/test_fused_qk_rms_norm.py diff --git a/tests/dreamzero/test_pipeline_state.py b/tests/diffusion/models/dreamzero/test_pipeline_state.py similarity index 100% rename from tests/dreamzero/test_pipeline_state.py rename to tests/diffusion/models/dreamzero/test_pipeline_state.py diff --git a/tests/dreamzero/test_qkv_fusion.py b/tests/diffusion/models/dreamzero/test_qkv_fusion.py similarity index 100% rename from tests/dreamzero/test_qkv_fusion.py rename to tests/diffusion/models/dreamzero/test_qkv_fusion.py diff --git a/tests/dreamzero/test_session_state_accounting.py b/tests/diffusion/models/dreamzero/test_session_state_accounting.py similarity index 100% rename from tests/dreamzero/test_session_state_accounting.py rename to tests/diffusion/models/dreamzero/test_session_state_accounting.py diff --git a/tests/dreamzero/test_session_state_equivalence.py b/tests/diffusion/models/dreamzero/test_session_state_equivalence.py similarity index 100% rename from tests/dreamzero/test_session_state_equivalence.py rename to tests/diffusion/models/dreamzero/test_session_state_equivalence.py diff --git a/tests/dreamzero/test_session_state_lifecycle.py b/tests/diffusion/models/dreamzero/test_session_state_lifecycle.py similarity index 100% rename from tests/dreamzero/test_session_state_lifecycle.py rename to tests/diffusion/models/dreamzero/test_session_state_lifecycle.py diff --git a/tests/diffusion/models/ltx2/test_ltx2_transformer.py b/tests/diffusion/models/ltx2/test_ltx2_transformer.py index de037fef8..73e10463e 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_transformer.py +++ b/tests/diffusion/models/ltx2/test_ltx2_transformer.py @@ -9,6 +9,8 @@ from vllm_omni.diffusion.models.ltx2.ltx2_transformer import LTX2VideoTransforme pytestmark = [pytest.mark.core_model, pytest.mark.cpu] +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + def test_ltx_rms_norm_no_affine_matches_official_torch_module(): norm = _make_rms_norm(8, eps=1e-6, elementwise_affine=False) diff --git a/tests/diffusion/models/sensenova_u1/test_sensenova_u1_cfg.py b/tests/diffusion/models/sensenova_u1/test_sensenova_u1_cfg.py index 818d3d7c0..276e9e059 100644 --- a/tests/diffusion/models/sensenova_u1/test_sensenova_u1_cfg.py +++ b/tests/diffusion/models/sensenova_u1/test_sensenova_u1_cfg.py @@ -6,6 +6,8 @@ import torch from vllm_omni.diffusion.models.sensenova_u1.pipeline_sensenova_u1 import SenseNovaU1Pipeline +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + def _pipeline_without_init() -> SenseNovaU1Pipeline: return object.__new__(SenseNovaU1Pipeline) diff --git a/tests/diffusion/offloader/test_diffusion_cpu_offload.py b/tests/diffusion/offloader/test_diffusion_cpu_offload.py index 555a62ca0..549a98142 100644 --- a/tests/diffusion/offloader/test_diffusion_cpu_offload.py +++ b/tests/diffusion/offloader/test_diffusion_cpu_offload.py @@ -5,7 +5,6 @@ import pytest import torch from vllm.distributed.parallel_state import cleanup_dist_env_and_memory -from tests.helpers import skip_if_gated_repo_inaccessible from tests.helpers.env import DeviceMemoryMonitor from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniRunner @@ -16,32 +15,26 @@ AUDIO_MODEL = { "stabilityai/stable-audio-open-1.0": {"cuda": 100, "rocm": None}, } -IMAGE_VIDEO_MODELS = { +IMAGE_MODELS = { "riverclouds/qwen_image_random": {"cuda": 2200, "rocm": 2100}, - "Tongyi-MAI/Z-Image-Turbo": {"cuda": 2500, "rocm": 2100}, - "OmniGen2/OmniGen2": {"cuda": 2500, "rocm": 2100}, } -MODELS = {**AUDIO_MODEL, **IMAGE_VIDEO_MODELS} +MODELS = {**AUDIO_MODEL, **IMAGE_MODELS} MODEL_MARKS = { "riverclouds/qwen_image_random": pytest.mark.core_model, - "Tongyi-MAI/Z-Image-Turbo": pytest.mark.advanced_model, "stabilityai/stable-audio-open-1.0": pytest.mark.full_model, - "OmniGen2/OmniGen2": pytest.mark.full_model, } _GATED_MODELS = {"stabilityai/stable-audio-open-1.0"} -# Aliased for backward compatibility (imported by test_diffusion_layerwise_offload.py). -_skip_if_gated_repo_inaccessible = skip_if_gated_repo_inaccessible AUDIO_MODEL_PARAMS = { "runner_params": {}, "sampler_params": {}, } -IMAGE_VIDEO_MODELS_PARAMS = { +IMAGE_MODEL_PARAMS = { "runner_params": {}, "sampler_params": { "height": 256, @@ -60,7 +53,7 @@ def inference(model_name: str, offload: bool = True): if model_name in AUDIO_MODEL: params = AUDIO_MODEL_PARAMS else: - params = IMAGE_VIDEO_MODELS_PARAMS + params = IMAGE_MODEL_PARAMS with OmniRunner( model_name, @@ -111,19 +104,12 @@ def check_audio_determinism(audio1, audio2, atol=1e-2): [pytest.param(name, marks=MODEL_MARKS[name]) for name in MODELS], ) def test_cpu_offload_diffusion_model(model_name: str): - if model_name == "OmniGen2/OmniGen2": - pytest.skip("issue #4537") - if model_name in _GATED_MODELS: - _skip_if_gated_repo_inaccessible(model_name) try: offload_peak_memory, output_offload = inference(model_name, offload=True) cleanup_dist_env_and_memory() no_offload_peak_memory, output_no_offload = inference(model_name, offload=False) except ValueError as exc: - # omni_snapshot_download wraps GatedRepoError in a ValueError. - # If the pre-flight guard above did not catch it (e.g. partial - # HF_TOKEN where config.json is accessible but weight shards are - # blocked), skip instead of failing. + # omni_snapshot_download wraps GatedRepoError in a ValueError; skip instead of failing. if "Access to model" in str(exc) and "is restricted" in str(exc): pytest.skip( f"Skipping: gated HF repo {model_name!r} inaccessible " diff --git a/tests/diffusion/offloader/test_diffusion_layerwise_offload.py b/tests/diffusion/offloader/test_diffusion_layerwise_offload.py index f02da6fab..342c9833b 100644 --- a/tests/diffusion/offloader/test_diffusion_layerwise_offload.py +++ b/tests/diffusion/offloader/test_diffusion_layerwise_offload.py @@ -3,11 +3,8 @@ import pytest import torch from vllm.distributed.parallel_state import cleanup_dist_env_and_memory -from tests.diffusion.offloader.test_diffusion_cpu_offload import ( - _GATED_MODELS, - _skip_if_gated_repo_inaccessible, -) from tests.helpers.env import DeviceMemoryMonitor +from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniRunner from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.platforms import current_omni_platform @@ -23,6 +20,19 @@ IMAGE_VIDEO_MODELS = { MODELS = {**AUDIO_MODEL, **IMAGE_VIDEO_MODELS} +LAYERWISE_MODEL_CASES = [ + pytest.param( + "stabilityai/stable-audio-open-1.0", + marks=[pytest.mark.full_model, pytest.mark.diffusion], + id="stable_audio", + ), + pytest.param( + "riverclouds/qwen_image_random", + marks=[pytest.mark.core_model, pytest.mark.diffusion], + id="qwen_image_random", + ), +] + AUDIO_MODEL_PARAMS = { "runner_params": {}, "sampler_params": {}, @@ -91,7 +101,8 @@ def run_inference( return peak, output -@pytest.mark.parametrize("model_name", list(MODELS.keys())) +@hardware_test(res={"cuda": "L4", "rocm": "MI325"}) +@pytest.mark.parametrize("model_name", LAYERWISE_MODEL_CASES) def test_layerwise_offload_diffusion_model(model_name: str): """Test that layerwise offloading reduces GPU memory usage. @@ -100,8 +111,6 @@ def test_layerwise_offload_diffusion_model(model_name: str): offloader keeps only a single transformer block on GPU at a time, with prefetching for compute-memory overlap. """ - if model_name in _GATED_MODELS: - _skip_if_gated_repo_inaccessible(model_name) try: # Run without layerwise offloading (baseline) no_offload_peak_memory, output_no_offload = run_inference(model_name, layerwise_offload=False) @@ -111,10 +120,7 @@ def test_layerwise_offload_diffusion_model(model_name: str): layerwise_offload_peak_memory, output_offload = run_inference(model_name, layerwise_offload=True) cleanup_dist_env_and_memory() except ValueError as exc: - # omni_snapshot_download wraps GatedRepoError in a ValueError. - # If the pre-flight guard above did not catch it (e.g. partial - # HF_TOKEN where config.json is accessible but weight shards are - # blocked), skip instead of failing. + # omni_snapshot_download wraps GatedRepoError in a ValueError; skip instead of failing. if "Access to model" in str(exc) and "is restricted" in str(exc): pytest.skip( f"Skipping: gated HF repo {model_name!r} inaccessible " diff --git a/tests/plugin/test_gguf_diffusion.py b/tests/diffusion/quantization/test_gguf_diffusion.py similarity index 97% rename from tests/plugin/test_gguf_diffusion.py rename to tests/diffusion/quantization/test_gguf_diffusion.py index 44de4a1fb..fa87e8a7c 100644 --- a/tests/plugin/test_gguf_diffusion.py +++ b/tests/diffusion/quantization/test_gguf_diffusion.py @@ -5,10 +5,10 @@ Validates that GGUF-quantized diffusion models generate valid images and use less peak GPU memory than BF16 baseline. -Requires vllm-omni to be installed alongside the plugin. +Requires vllm-omni to be installed alongside the vllm-gguf-plugin package. Usage: - pytest tests/diffusion/test_gguf_memory.py -v + pytest tests/diffusion/quantization/test_gguf_diffusion.py -v """ from __future__ import annotations @@ -112,7 +112,6 @@ def _generate_single_stage_image( @hardware_test(res={"cuda": "L4"}) @pytest.mark.full_model @pytest.mark.diffusion -@pytest.mark.slow @pytest.mark.parametrize("model", [Z_IMAGE_CONFIG, FLUX_CONFIG], ids=["Z-Image-Turbo", "FLUX.2-klein"]) def test_single_stage_diffusion_gguf(model: DiffusionGGUFTestConfig, monkeypatch: pytest.MonkeyPatch) -> None: """Z-Image-Turbo GGUF generates valid images and uses less memory than BF16.""" diff --git a/tests/diffusion/quantization/test_quantization_quality.py b/tests/diffusion/quantization/test_quantization_quality.py index c2807797d..d3cee6b6a 100644 --- a/tests/diffusion/quantization/test_quantization_quality.py +++ b/tests/diffusion/quantization/test_quantization_quality.py @@ -431,6 +431,8 @@ def _free_gpu_memory(): # --------------------------------------------------------------------------- +@pytest.mark.core_model +@pytest.mark.cpu def test_benchmark_generate_image_unwraps_nested_omni_request_output(monkeypatch): from benchmarks.diffusion.quantization_quality import _generate_image as benchmark_generate_image from vllm_omni.outputs import OmniRequestOutput diff --git a/tests/diffusion/quantization/test_trajectory_similarity_tool.py b/tests/diffusion/quantization/test_trajectory_similarity_tool.py index 252e6f1c1..963ed5685 100644 --- a/tests/diffusion/quantization/test_trajectory_similarity_tool.py +++ b/tests/diffusion/quantization/test_trajectory_similarity_tool.py @@ -22,6 +22,8 @@ from vllm_omni.quantization.tools.compare_diffusion_trajectory_similarity import summarize_output_image_metrics, ) +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + def test_compute_tensor_metrics_identical_tensors(): metrics = compute_tensor_metrics(torch.ones(2, 3), torch.ones(2, 3)) diff --git a/tests/test_diffusion_config_fields.py b/tests/diffusion/test_diffusion_config_fields.py similarity index 97% rename from tests/test_diffusion_config_fields.py rename to tests/diffusion/test_diffusion_config_fields.py index b87ceec1d..d62e7bdc8 100644 --- a/tests/test_diffusion_config_fields.py +++ b/tests/diffusion/test_diffusion_config_fields.py @@ -29,7 +29,7 @@ def test_diffusion_stage_configs_only_contain_valid_fields(): Regression test for https://github.com/vllm-project/vllm-omni/issues/2563 """ # Scan both main configs and test configs - repo_root = Path(__file__).parent.parent + repo_root = Path(__file__).parent.parent.parent config_dirs = [ repo_root / "vllm_omni" / "model_executor" / "stage_configs", ] diff --git a/tests/test_diffusion_config_propagation.py b/tests/diffusion/test_diffusion_config_propagation.py similarity index 100% rename from tests/test_diffusion_config_propagation.py rename to tests/diffusion/test_diffusion_config_propagation.py diff --git a/tests/diffusion/test_diffusion_engine.py b/tests/diffusion/test_diffusion_engine.py index e44766feb..192675908 100644 --- a/tests/diffusion/test_diffusion_engine.py +++ b/tests/diffusion/test_diffusion_engine.py @@ -14,6 +14,7 @@ import torch from pytest_mock import MockerFixture import vllm_omni.diffusion.diffusion_engine as diffusion_engine_module +from tests.helpers.mark import hardware_test from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig from vllm_omni.diffusion.diffusion_engine import ( DiffusionEngine, @@ -30,6 +31,8 @@ from vllm_omni.diffusion.sched.interface import ( ) from vllm_omni.inputs.data import OmniDiffusionSamplingParams +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion] + @dataclass class DiffusionSchedulerOutput: @@ -56,6 +59,8 @@ class MockScheduler: def __init__(self): self._waiting_queue = [] self._step_id = 0 + # Match RequestScheduler API used by request-batch admission wait. + self.max_num_running_reqs = 5 def add_request(self, request): self._waiting_queue.append(request) @@ -64,6 +69,12 @@ class MockScheduler: def has_requests(self): return len(self._waiting_queue) > 0 + def num_waiting_requests(self) -> int: + return len(self._waiting_queue) + + def num_running_requests(self) -> int: + return 0 + def schedule(self) -> DiffusionSchedulerOutput: if not self._waiting_queue: return DiffusionSchedulerOutput(step_id=self._step_id) @@ -582,9 +593,7 @@ def test_move_tensor_tree_returns_non_tensor_values_unchanged() -> None: assert moved is value -@pytest.mark.diffusion -@pytest.mark.cuda -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +@hardware_test(res={"cuda": "L4"}, num_cards=1) def test_move_tensor_tree_moves_nested_cuda_tensors_to_cpu() -> None: tensor = torch.arange(8, dtype=torch.float32, device="cuda") other = torch.arange(4, dtype=torch.int64, device="cuda") @@ -610,6 +619,7 @@ async def _consume_final_output(generator): return final_output +@pytest.mark.cpu @pytest.mark.asyncio async def test_async_add_req_and_stream_response(): engine = object.__new__(DiffusionEngine) @@ -621,13 +631,24 @@ async def test_async_add_req_and_stream_response(): engine._cv = threading.Condition(engine._rpc_lock) engine._init_lock = asyncio.Lock() engine._closed = False - engine.od_config = SimpleNamespace(streaming_output=False) + # Enable admission wait so concurrent adds land in one schedule wave; + # otherwise the first request can execute alone (~1s) and the rest in a + # second wave (~2s), failing the latency-spread assertion. + engine.od_config = SimpleNamespace( + streaming_output=False, + request_batch_max_wait_ms=500.0, + ) engine._loop_started = False engine.main_loop = None - engine.supports_request_batch = False + engine.supports_request_batch = True + engine.step_execution = False engine.execution_mode = DiffusionExecutionMode.REQUEST_BATCH - engine._finalize_finished_request = lambda rid, out, err: out.result + def _finalize(rid, out, err=None): + # Stream consumers stop on ``finished``; keep result_data for assertions. + return SimpleNamespace(result_data=out.result.result_data, finished=True) + + engine._finalize_finished_request = _finalize def mock_execute_batch(sched_output): request_ids = sched_output.scheduled_request_ids diff --git a/tests/diffusion/test_diffusion_engine_metrics.py b/tests/diffusion/test_diffusion_engine_metrics.py index 06ad2077f..8f434c0e2 100644 --- a/tests/diffusion/test_diffusion_engine_metrics.py +++ b/tests/diffusion/test_diffusion_engine_metrics.py @@ -12,6 +12,11 @@ from __future__ import annotations import ast import os +import pytest + +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + + _ENGINE_PATH = os.path.normpath( os.path.join( os.path.dirname(__file__), diff --git a/tests/docs/test_generate_examples.py b/tests/docs/test_generate_examples.py index 0b29837a9..57d7d4832 100644 --- a/tests/docs/test_generate_examples.py +++ b/tests/docs/test_generate_examples.py @@ -6,6 +6,9 @@ from pathlib import Path import pytest +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + ROOT_DIR = Path(__file__).parents[2] HOOK_PATH = ROOT_DIR / "docs/mkdocs/hooks/generate_examples.py" SPEC = importlib.util.spec_from_file_location("generate_examples", HOOK_PATH) diff --git a/tests/dreamzero/openpi_client_helper.py b/tests/dreamzero/openpi_client_helper.py deleted file mode 100644 index 8918e5c36..000000000 --- a/tests/dreamzero/openpi_client_helper.py +++ /dev/null @@ -1,307 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from __future__ import annotations - -import uuid -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -import numpy as np - -try: - import cv2 -except ImportError: # pragma: no cover - optional e2e dependency - cv2 = None - -try: - import websockets.sync.client as websockets_client -except ImportError: # pragma: no cover - optional e2e dependency - websockets_client = None - -try: - from openpi_client import msgpack_numpy -except ImportError: # pragma: no cover - optional e2e dependency - msgpack_numpy = None - -PING_INTERVAL_SECS = 300 -PING_TIMEOUT_SECS = 3600 -DEFAULT_HOST = "127.0.0.1" -DEFAULT_PORT = 8000 -DEFAULT_PATH = "/v1/realtime/robot/openpi" -DEFAULT_PROMPT = "Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan" -ACTION_HORIZON = 24 -DEFAULT_ACTION_DIM = 8 -RELATIVE_OFFSETS = [-23, -16, -8, 0] -CAMERA_FILES = { - "observation/exterior_image_0_left": "exterior_image_1_left.mp4", - "observation/exterior_image_1_left": "exterior_image_2_left.mp4", - "observation/wrist_image_left": "wrist_image_left.mp4", -} - - -def _decode_action_response(response: bytes | str) -> np.ndarray: - if isinstance(response, str): - raise RuntimeError(f"Inference failed: {response}") - decoded = msgpack_numpy.unpackb(response) - if isinstance(decoded, dict) and decoded.get("type") == "error": - message = decoded.get("message", decoded) - raise RuntimeError(f"Inference failed: {message}") - return np.asarray(decoded, dtype=np.float32) - - -def require_dependencies() -> None: - missing = [] - if cv2 is None: - missing.append("opencv-python") - if websockets_client is None: - missing.append("websockets") - if msgpack_numpy is None: - missing.append("openpi-client") - if missing: - raise ModuleNotFoundError(f"DreamZero OpenPI test dependencies are missing: {', '.join(missing)}") - - -@dataclass(frozen=True) -class DreamZeroServerMetadata: - image_resolution: tuple[int, int] - n_external_cameras: int - needs_wrist_camera: bool - needs_stereo_camera: bool - needs_session_id: bool - action_space: str - - @classmethod - def from_dict(cls, payload: dict[str, Any]) -> DreamZeroServerMetadata: - required_keys = ( - "image_resolution", - "n_external_cameras", - "needs_wrist_camera", - "needs_stereo_camera", - "needs_session_id", - "action_space", - ) - missing_keys = [key for key in required_keys if key not in payload] - if missing_keys: - raise ValueError(f"Missing DreamZero metadata keys: {missing_keys}") - - image_resolution = payload["image_resolution"] - if not isinstance(image_resolution, (list, tuple)) or len(image_resolution) != 2: - raise ValueError(f"Invalid image_resolution: {image_resolution!r}") - - return cls( - image_resolution=(int(image_resolution[0]), int(image_resolution[1])), - n_external_cameras=int(payload["n_external_cameras"]), - needs_wrist_camera=bool(payload["needs_wrist_camera"]), - needs_stereo_camera=bool(payload["needs_stereo_camera"]), - needs_session_id=bool(payload["needs_session_id"]), - action_space=str(payload["action_space"]), - ) - - -class OpenPIWebsocketClient: - def __init__( - self, - *, - host: str = DEFAULT_HOST, - port: int = DEFAULT_PORT, - path: str = DEFAULT_PATH, - ) -> None: - require_dependencies() - self._uri = f"ws://{host}:{port}{path}" - self._packer = msgpack_numpy.Packer() - self._ws, self._server_metadata = self._connect() - - def _connect(self): - conn = websockets_client.connect( - self._uri, - compression=None, - max_size=None, - ping_interval=PING_INTERVAL_SECS, - ping_timeout=PING_TIMEOUT_SECS, - ) - metadata = msgpack_numpy.unpackb(conn.recv()) - if not isinstance(metadata, dict): - raise TypeError(f"Expected dict metadata from server, got {type(metadata)!r}") - return conn, metadata - - def get_server_metadata(self) -> dict[str, Any]: - return dict(self._server_metadata) - - def infer(self, obs: dict[str, Any]) -> np.ndarray: - payload = dict(obs) - payload["endpoint"] = "infer" - self._ws.send(self._packer.pack(payload)) - response = self._ws.recv() - return _decode_action_response(response) - - def reset(self, reset_info: dict[str, Any] | None = None) -> str: - payload = dict(reset_info or {}) - payload["endpoint"] = "reset" - self._ws.send(self._packer.pack(payload)) - response = self._ws.recv() - if isinstance(response, str): - return response - decoded = msgpack_numpy.unpackb(response) - if not isinstance(decoded, dict) or decoded.get("status") != "reset successful": - raise RuntimeError(f"Unexpected reset response: {decoded!r}") - return str(decoded["status"]) - - def close(self) -> None: - self._ws.close() - - -def load_all_frames(video_path: Path) -> np.ndarray: - require_dependencies() - cap = cv2.VideoCapture(str(video_path)) - frames = [] - while True: - ok, frame = cap.read() - if not ok: - break - frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) - cap.release() - if not frames: - raise RuntimeError(f"No frames loaded from {video_path}") - return np.stack(frames, axis=0) - - -def load_camera_frames(video_dir: Path) -> dict[str, np.ndarray]: - camera_frames: dict[str, np.ndarray] = {} - for camera_key, file_name in CAMERA_FILES.items(): - video_path = video_dir / file_name - if not video_path.exists(): - raise FileNotFoundError(f"Missing DreamZero test asset: {video_path}") - camera_frames[camera_key] = load_all_frames(video_path) - return camera_frames - - -def build_frame_schedule(total_frames: int, num_chunks: int) -> list[list[int]]: - chunks: list[list[int]] = [] - current_frame = 23 - for _ in range(num_chunks): - indices = [max(current_frame + offset, 0) for offset in RELATIVE_OFFSETS] - if indices[-1] >= total_frames: - break - chunks.append(indices) - current_frame += ACTION_HORIZON - return chunks - - -def make_obs_from_video( - camera_frames: dict[str, np.ndarray], - frame_indices: list[int], - *, - prompt: str, - session_id: str, -) -> dict[str, Any]: - obs: dict[str, Any] = {} - for camera_key, all_frames in camera_frames.items(): - selected = all_frames[frame_indices] - obs[camera_key] = selected[0] if len(frame_indices) == 1 else selected - - obs["observation/joint_position"] = np.zeros(7, dtype=np.float32) - obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32) - obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32) - obs["prompt"] = prompt - obs["session_id"] = session_id - return obs - - -def build_demo_observations( - camera_frames: dict[str, np.ndarray], - *, - prompt: str, - session_id: str, - num_chunks: int = 2, -) -> list[dict[str, Any]]: - if num_chunks < 1: - raise ValueError("num_chunks must be at least 1") - - total_frames = min(frames.shape[0] for frames in camera_frames.values()) - observations = [ - make_obs_from_video( - camera_frames, - [0], - prompt=prompt, - session_id=session_id, - ) - ] - for indices in build_frame_schedule(total_frames, num_chunks - 1): - observations.append( - make_obs_from_video( - camera_frames, - indices, - prompt=prompt, - session_id=session_id, - ) - ) - return observations - - -def validate_session_result( - result: dict[str, Any], - *, - expected_action_horizon: int = ACTION_HORIZON, - expected_action_dim: int = DEFAULT_ACTION_DIM, -) -> None: - metadata = DreamZeroServerMetadata.from_dict(result["metadata"]) - if metadata.image_resolution != (180, 320): - raise AssertionError(f"Unexpected image_resolution: {metadata.image_resolution}") - if metadata.n_external_cameras != 2: - raise AssertionError(f"Unexpected n_external_cameras: {metadata.n_external_cameras}") - if not metadata.needs_wrist_camera: - raise AssertionError("DreamZero test expects wrist camera metadata") - if metadata.action_space != "joint_position": - raise AssertionError(f"Unexpected action_space: {metadata.action_space}") - - actions = result["actions"] - if len(actions) != 3: - raise AssertionError(f"Expected 3 action tensors, got {len(actions)}") - for index, action in enumerate(actions): - if action.shape != (expected_action_horizon, expected_action_dim): - raise AssertionError( - f"Action {index} shape mismatch: expected " - f"{(expected_action_horizon, expected_action_dim)}, got {action.shape}" - ) - if not np.isfinite(action).all(): - raise AssertionError(f"Action {index} contains non-finite values") - - if result["reset_status"] != "reset successful": - raise AssertionError(f"Unexpected reset status: {result['reset_status']!r}") - - -def run_policy_session( - *, - host: str = DEFAULT_HOST, - port: int = DEFAULT_PORT, - path: str = DEFAULT_PATH, - video_dir: Path, - prompt: str = DEFAULT_PROMPT, - session_id: str | None = None, - num_chunks: int = 2, -) -> dict[str, Any]: - session_id = session_id or str(uuid.uuid4()) - camera_frames = load_camera_frames(video_dir) - observations = build_demo_observations( - camera_frames, - prompt=prompt, - session_id=session_id, - num_chunks=num_chunks, - ) - - client = OpenPIWebsocketClient(host=host, port=port, path=path) - try: - metadata = client.get_server_metadata() - actions = [client.infer(obs) for obs in observations] - reset_status = client.reset({}) - actions.append(client.infer(observations[0])) - return { - "metadata": metadata, - "actions": actions, - "reset_status": reset_status, - "session_id": session_id, - } - finally: - client.close() diff --git a/tests/dreamzero/test_openpi_client_helper.py b/tests/dreamzero/test_openpi_client_helper.py deleted file mode 100644 index 76950a9ed..000000000 --- a/tests/dreamzero/test_openpi_client_helper.py +++ /dev/null @@ -1,46 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import numpy as np -import pytest - -from tests.dreamzero import openpi_client_helper - -pytestmark = [pytest.mark.core_model, pytest.mark.cpu] - - -class FakeMsgpackNumpy: - payloads = {} - - @staticmethod - def packb(obj): - key = str(len(FakeMsgpackNumpy.payloads)).encode() - FakeMsgpackNumpy.payloads[key] = obj - return key - - @staticmethod - def unpackb(data): - return FakeMsgpackNumpy.payloads[data] - - -def test_decode_action_response_surfaces_structured_error(monkeypatch): - monkeypatch.setattr(openpi_client_helper, "msgpack_numpy", FakeMsgpackNumpy) - payload = FakeMsgpackNumpy.packb( - { - "type": "error", - "message": "Internal inference error", - } - ) - - with pytest.raises(RuntimeError, match="Internal inference error"): - openpi_client_helper._decode_action_response(payload) - - -def test_decode_action_response_converts_action_payload_to_float32(monkeypatch): - monkeypatch.setattr(openpi_client_helper, "msgpack_numpy", FakeMsgpackNumpy) - payload = FakeMsgpackNumpy.packb(np.asarray([[1.0, 2.0]], dtype=np.float64)) - - actions = openpi_client_helper._decode_action_response(payload) - - assert actions.dtype == np.float32 - np.testing.assert_array_equal(actions, np.asarray([[1.0, 2.0]], dtype=np.float32)) diff --git a/tests/dreamzero/upstream/openpi_test_client_ar.py b/tests/dreamzero/upstream/openpi_test_client_ar.py deleted file mode 100644 index 7c5aea1d9..000000000 --- a/tests/dreamzero/upstream/openpi_test_client_ar.py +++ /dev/null @@ -1,381 +0,0 @@ -#!/usr/bin/env python3 -""" -Copied from the DreamZero repository's `test_client_AR.py`: -https://github.com/dreamzero0/dreamzero/blob/main/test_client_AR.py - -Kept here for end-to-end compatibility / parity testing against the vLLM -OpenPI server. - -Test client for AR_droid policy server using roboarena interface. - -Sends real video frames from debug_image/ directory instead of zero dummy images. - -Frame schedule (matching debug_inference.py): - - Step 0 (initial): send frame [0] (1 frame, H W 3) - - Step 1: send frames [0, 7, 15, 23] (4 frames, 4 H W 3) - - Step 2: send frames [24, 31, 39, 47] (4 frames) - - Step 3: send frames [48, 55, 63, 71] (4 frames) - - ... - -Expected server configuration: - - image_resolution: (180, 320) - - n_external_cameras: 2 - - needs_wrist_camera: True - - action_space: "joint_position" - -Usage: - # Start server with roboarena interface: - torchrun --nproc_per_node=8 socket_test_optimized_AR.py --port 8000 - - # Run against the original DreamZero websocket server (default path): - python test_client_AR.py --host --port 8000 - - # Run against the vLLM OpenPI server: - python test_client_AR.py --host --port 8000 --path /v1/realtime/robot/openpi - - # Use zero images instead of real video (old behavior): - python test_client_AR.py --host --port 8000 --use-zero-images -""" - -import argparse -import logging -import os -import sys -import time -import uuid -from pathlib import Path - -import cv2 -import numpy as np -from openpi_client import msgpack_numpy - -_DREAMZERO_REPO_ENV = os.environ.get("DREAMZERO_REPO") -DREAMZERO_REPO = Path(_DREAMZERO_REPO_ENV).expanduser() if _DREAMZERO_REPO_ENV else None - - -def _import_upstream_policy_modules(): - if DREAMZERO_REPO is None: - raise ImportError("Set DREAMZERO_REPO to an upstream DreamZero checkout before using this helper.") - if DREAMZERO_REPO.exists() and str(DREAMZERO_REPO) not in sys.path: - sys.path.insert(0, str(DREAMZERO_REPO)) - - import eval_utils.policy_server as policy_server - from eval_utils.policy_client import WebsocketClientPolicy - - return policy_server, WebsocketClientPolicy - - -policy_server, WebsocketClientPolicy = _import_upstream_policy_modules() - -VIDEO_DIR = os.environ.get( - "DREAMZERO_VIDEO_DIR", - str(DREAMZERO_REPO / "debug_image") if DREAMZERO_REPO is not None else "debug_image", -) - -# roboarena key -> video filename -CAMERA_FILES = { - "observation/exterior_image_0_left": "exterior_image_1_left.mp4", - "observation/exterior_image_1_left": "exterior_image_2_left.mp4", - "observation/wrist_image_left": "wrist_image_left.mp4", -} - -# Frame schedule constants (matching debug_inference.py) -RELATIVE_OFFSETS = [-23, -16, -8, 0] -ACTION_HORIZON = 24 -DEFAULT_WEBSOCKET_PATH = "" - - -class OpenPIWebsocketClientPolicy(WebsocketClientPolicy): - """DreamZero websocket client with a configurable path suffix. - - The original DreamZero client connects to ``ws://host:port``. - vLLM serves the compatible robot policy endpoint at - ``/v1/realtime/robot/openpi`` when ``path`` is set accordingly. - """ - - def __init__( - self, - host: str = "0.0.0.0", - port: int = 8000, - path: str = DEFAULT_WEBSOCKET_PATH, - ) -> None: - self._uri = f"ws://{host}:{port}{path}" - self._packer = msgpack_numpy.Packer() - self._ws, self._server_metadata = self._wait_for_server() - - -def load_all_frames(video_path: str) -> np.ndarray: - """Load all frames from a video file. Returns (N, H, W, 3) uint8 array (RGB).""" - cap = cv2.VideoCapture(video_path) - frames = [] - while True: - ret, frame = cap.read() - if not ret: - break - frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) - cap.release() - if not frames: - raise RuntimeError(f"No frames loaded from {video_path}") - return np.stack(frames, axis=0) - - -def load_camera_frames() -> dict[str, np.ndarray]: - """Load all video frames for each camera from the debug_image/ directory. - - Returns: - Dict mapping roboarena camera keys to (N, H, W, 3) uint8 arrays. - """ - camera_frames: dict[str, np.ndarray] = {} - for cam_key, fname in CAMERA_FILES.items(): - path = os.path.join(VIDEO_DIR, fname) - camera_frames[cam_key] = load_all_frames(path) - logging.info(f"Loaded {cam_key}: {camera_frames[cam_key].shape}") - return camera_frames - - -def build_frame_schedule(total_frames: int, num_chunks: int) -> list[list[int]]: - """Build the frame index schedule for multi-frame chunks. - - Returns a list of frame-index lists. Each inner list has 4 indices. - """ - chunks: list[list[int]] = [] - current_frame = 23 # first anchor frame - for _ in range(num_chunks): - indices = [max(current_frame + off, 0) for off in RELATIVE_OFFSETS] - if indices[-1] >= total_frames: - logging.info(f"Frame {indices[-1]} >= {total_frames}, stopping at {len(chunks)} chunks") - break - chunks.append(indices) - current_frame += ACTION_HORIZON - return chunks - - -def _make_obs_from_video( - camera_frames: dict[str, np.ndarray], - frame_indices: list[int], - prompt: str, - session_id: str, -) -> dict: - """Build an observation dict from real video frames. - - For 1 frame: each image key is (H, W, 3). - For 4 frames: each image key is (4, H, W, 3). - """ - obs: dict = {} - for cam_key, all_frames in camera_frames.items(): - selected = all_frames[frame_indices] # (T, H, W, 3) - if len(frame_indices) == 1: - selected = selected[0] # (H, W, 3) - obs[cam_key] = selected - - obs["observation/joint_position"] = np.zeros(7, dtype=np.float32) - obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32) - obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32) - obs["prompt"] = prompt - obs["session_id"] = session_id - return obs - - -def _make_zero_observation( - server_config: policy_server.PolicyServerConfig, - prompt: str = "pick up the object", - session_id: str | None = None, -) -> dict: - """Create a dummy observation matching AR_droid expectations. - - AR_droid expects: - - 2 external cameras (exterior_image_0_left, exterior_image_1_left) - - 1 wrist camera (wrist_image_left) - - Image resolution: 180x320 (H x W) - - joint_position: 7 DoF - - gripper_position: 1 DoF - """ - obs = {} - - # Determine image resolution - if server_config.image_resolution is not None: - h, w = server_config.image_resolution - else: - # Default for AR_droid - h, w = 180, 320 - - # External cameras (0-indexed in roboarena) - for i in range(server_config.n_external_cameras): - obs[f"observation/exterior_image_{i}_left"] = np.zeros((h, w, 3), dtype=np.uint8) - if server_config.needs_stereo_camera: - obs[f"observation/exterior_image_{i}_right"] = np.zeros((h, w, 3), dtype=np.uint8) - - # Wrist camera - if server_config.needs_wrist_camera: - obs["observation/wrist_image_left"] = np.zeros((h, w, 3), dtype=np.uint8) - if server_config.needs_stereo_camera: - obs["observation/wrist_image_right"] = np.zeros((h, w, 3), dtype=np.uint8) - - # Session ID - should be passed in to ensure consistency within a session - if server_config.needs_session_id: - import uuid - - # Generate unique session ID if not provided - obs["session_id"] = session_id if session_id else str(uuid.uuid4()) - - # State observations (AR_droid: 7 DoF arm + 1 gripper) - obs["observation/joint_position"] = np.zeros(7, dtype=np.float32) - obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32) - obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32) - - # Language prompt - obs["prompt"] = prompt - - return obs - - -def test_ar_droid_policy_server( - host: str = "localhost", - port: int = 8000, - path: str = DEFAULT_WEBSOCKET_PATH, - num_chunks: int = 15, - prompt: str = "Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan", - use_zero_images: bool = False, -): - """Test the AR_droid policy server with roboarena interface. - - When use_zero_images is False (default), loads real video frames from - debug_image/ and follows the frame schedule from debug_inference.py. - """ - logging.info(f"Connecting to AR_droid server at ws://{host}:{port}{path} ...") - - client = OpenPIWebsocketClientPolicy(host=host, port=port, path=path) - - # Validate server metadata - metadata = client.get_server_metadata() - logging.info(f"Server metadata: {metadata}") - assert isinstance(metadata, dict), "Metadata should be a dict" - - try: - server_config = policy_server.PolicyServerConfig(**metadata) - except Exception as e: - logging.error(f"Error parsing metadata: {e}") - raise e - - # Validate expected AR_droid configuration - logging.info(f"Server config: {server_config}") - assert server_config.n_external_cameras == 2, f"Expected 2 external cameras, got {server_config.n_external_cameras}" - assert server_config.needs_wrist_camera, "Expected wrist camera to be enabled" - assert server_config.action_space == "joint_position", ( - f"Expected joint_position action space, got {server_config.action_space}" - ) - - logging.info("Server configuration validated for AR_droid") - - # Generate unique session ID for this test run - session_id = str(uuid.uuid4()) - logging.info(f"Session ID: {session_id}") - - # ── Zero-image fallback mode ────────────────────────────────────── - if use_zero_images: - logging.info("Using ZERO dummy images (legacy mode)") - for i in range(num_chunks): - obs = _make_zero_observation(server_config, prompt=prompt, session_id=session_id) - logging.info(f"Inference {i + 1}/{num_chunks}: prompt='{prompt}'") - t0 = time.time() - actions = client.infer(obs) - dt = time.time() - t0 - _log_action(actions, dt) - - logging.info("Sending reset...") - client.reset({}) - logging.info("Done (zero-image mode).") - return - - # ── Real video frame mode ───────────────────────────────────────── - logging.info("Loading real video frames from debug_image/ directory") - camera_frames = load_camera_frames() - - total_frames = min(v.shape[0] for v in camera_frames.values()) - logging.info(f"Total frames available: {total_frames}") - - # Build frame schedule - chunks = build_frame_schedule(total_frames, num_chunks) - - logging.info("Frame schedule:") - logging.info(" Initial: [0]") - for i, indices in enumerate(chunks): - logging.info(f" Chunk {i}: {indices}") - - # Step 0: initial single frame - logging.info("=== Initial: frame [0] ===") - obs = _make_obs_from_video(camera_frames, [0], prompt, session_id) - t0 = time.time() - actions = client.infer(obs) - dt = time.time() - t0 - _log_action(actions, dt) - - # Subsequent chunks: send 4 frames at a time - for chunk_idx, frame_indices in enumerate(chunks): - logging.info(f"=== Chunk {chunk_idx}: frames {frame_indices} ===") - obs = _make_obs_from_video(camera_frames, frame_indices, prompt, session_id) - t0 = time.time() - actions = client.infer(obs) - dt = time.time() - t0 - _log_action(actions, dt) - - # Reset triggers video save on the server - logging.info("Sending reset to save video...") - client.reset({}) - - logging.info("Done.") - - -def _log_action(actions: np.ndarray, dt: float) -> None: - """Pretty-print action shape, range, and timing.""" - assert isinstance(actions, np.ndarray), f"Expected numpy array, got {type(actions)}" - assert actions.ndim == 2, f"Expected 2D array, got shape {actions.shape}" - assert actions.shape[-1] == 8, f"Expected 8 action dims (7 joints + 1 gripper), got {actions.shape[-1]}" - logging.info(f" Action shape: {actions.shape}, range: [{actions.min():.4f}, {actions.max():.4f}], time: {dt:.2f}s") - - -def main(): - parser = argparse.ArgumentParser(description="Test AR_droid policy server with real video frames from debug_image/") - parser.add_argument("--host", default="localhost", help="Server hostname") - parser.add_argument("--port", type=int, default=8000, help="Server port") - parser.add_argument( - "--path", - default=DEFAULT_WEBSOCKET_PATH, - help="WebSocket path suffix (default: empty string for the original DreamZero server)", - ) - parser.add_argument( - "--num-chunks", - type=int, - default=15, - help="Number of 4-frame chunks to send after the initial frame (default: 15)", - ) - parser.add_argument( - "--prompt", - default="Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan", - help="Language prompt for the policy", - ) - parser.add_argument( - "--use-zero-images", - action="store_true", - help="Use zero dummy images instead of real video frames (legacy mode)", - ) - - args = parser.parse_args() - - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s [%(levelname)s] %(message)s", - ) - - test_ar_droid_policy_server( - host=args.host, - port=args.port, - path=args.path, - num_chunks=args.num_chunks, - prompt=args.prompt, - use_zero_images=args.use_zero_images, - ) - - -if __name__ == "__main__": - main() diff --git a/tests/dreamzero/upstream/test_openpi_e2e_source_parity.py b/tests/dreamzero/upstream/test_openpi_e2e_source_parity.py deleted file mode 100644 index e41cff903..000000000 --- a/tests/dreamzero/upstream/test_openpi_e2e_source_parity.py +++ /dev/null @@ -1,358 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -"""Formal OpenPI end-to-end parity: upstream DreamZero server vs `vllm serve`. - -This test uses DreamZero's own client-side observation builders from -`${DREAMZERO_REPO}/test_client_AR.py`, and client-side websocket protocol from -`${DREAMZERO_REPO}/eval_utils/policy_client.py`. - -The only client-side adaptation for vLLM is the websocket path: -DreamZero's upstream server serves at `/`, while vLLM serves OpenPI at -`/v1/realtime/robot/openpi`. - -Current scope for this test: -- default two-GPU run (`nproc_per_node=2` on upstream, `--cfg-parallel-size 2` on `vllm serve`) -- non-`torch.compile` (upstream launched through - `upstream_socket_server_no_compile.py`, vLLM with `--enforce-eager`) -- non-DiT-cache / non-skip-schedule (`NUM_DIT_STEPS=16`) - -Serving contract locked by this test: -- upstream DreamZero still boots from the local checkpoint directory -- vLLM boots from the official DreamZero HF repo name (`GEAR-Dreams/DreamZero-DROID`) - rather than a prepared local bundle path -""" - -from __future__ import annotations - -import os -import shutil -import subprocess -import sys -import time -from pathlib import Path - -import numpy as np -import pytest -import torch - -from tests.helpers.runtime import get_open_port - -msgpack_numpy = pytest.importorskip("openpi_client.msgpack_numpy") - -_DREAMZERO_REPO_ENV = os.environ.get("DREAMZERO_REPO") -DREAMZERO_REPO = Path(_DREAMZERO_REPO_ENV).expanduser() if _DREAMZERO_REPO_ENV else None -if DREAMZERO_REPO is not None and str(DREAMZERO_REPO) not in sys.path: - sys.path.insert(0, str(DREAMZERO_REPO)) - -try: - import test_client_AR as dreamzero_client - from eval_utils.policy_client import WebsocketClientPolicy -except Exception: # pragma: no cover - guarded by pytest skip below - dreamzero_client = None - WebsocketClientPolicy = None -_BaseWebsocketClientPolicy = WebsocketClientPolicy if WebsocketClientPolicy is not None else object - -CHECKPOINT_DIR = DREAMZERO_REPO / "checkpoints" / "dreamzero" if DREAMZERO_REPO is not None else None -VLLM_MODEL = os.environ.get("VLLM_DREAMZERO_MODEL", "GEAR-Dreams/DreamZero-DROID") -SERVICE_READY_TIMEOUT_S = int(os.environ.get("OPENPI_SERVICE_READY_TIMEOUT_S", "900")) -PROMPT = "Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan" -SESSION_ID = "openpi-e2e-parity-session" - -pytestmark = [ - pytest.mark.skipif(not torch.cuda.is_available(), reason="GPU required"), - pytest.mark.skipif( - dreamzero_client is None or WebsocketClientPolicy is None, - reason="DreamZero client modules are required on PYTHONPATH", - ), - pytest.mark.skipif( - DREAMZERO_REPO is None or not DREAMZERO_REPO.exists(), - reason="DreamZero source repo is required at DREAMZERO_REPO", - ), - pytest.mark.skipif( - CHECKPOINT_DIR is None or not CHECKPOINT_DIR.exists(), reason="DreamZero local checkpoint is required" - ), -] - - -class OpenPIWebsocketClientPolicy(_BaseWebsocketClientPolicy): - """DreamZero client protocol with an OpenPI websocket path suffix.""" - - def __init__( - self, - host: str = "127.0.0.1", - port: int = 8000, - path: str = "/v1/realtime/robot/openpi", - ) -> None: - self._uri = f"ws://{host}:{port}{path}" - self._packer = msgpack_numpy.Packer() - self._ws, self._server_metadata = self._wait_for_server() - - -def _vllm_executable() -> str: - fallback = Path(sys.executable).with_name("vllm") - if fallback.exists(): - return str(fallback) - exe = shutil.which("vllm") - if exe: - return exe - raise FileNotFoundError("Unable to locate `vllm` executable in current environment.") - - -def _cfg_parallel_size() -> int: - return int(os.environ.get("OPENPI_E2E_CFG_PARALLEL_SIZE", "2")) - - -def _pick_test_gpus() -> list[str]: - cfg_parallel_size = _cfg_parallel_size() - override = os.environ.get("OPENPI_E2E_GPUS") or os.environ.get("OPENPI_E2E_GPU") - if override is not None: - gpus = [part.strip() for part in override.split(",") if part.strip()] - if not gpus: - raise ValueError("OPENPI_E2E_GPUS is set but empty.") - if len(gpus) < cfg_parallel_size: - raise RuntimeError(f"Need {cfg_parallel_size} GPUs, but OPENPI_E2E_GPUS only provided {gpus}.") - return gpus - - query = subprocess.check_output( - [ - "nvidia-smi", - "--query-gpu=index,memory.used", - "--format=csv,noheader,nounits", - ], - text=True, - ) - gpu_rows = [] - for line in query.strip().splitlines(): - gpu_index, used_mb = [part.strip() for part in line.split(",", maxsplit=1)] - gpu_rows.append((int(used_mb), gpu_index)) - gpu_rows.sort() - gpus = [gpu_index for _, gpu_index in gpu_rows[: max(cfg_parallel_size, 1)]] - if len(gpus) < cfg_parallel_size: - raise RuntimeError( - f"Need {cfg_parallel_size} GPUs for cfg_parallel_size={cfg_parallel_size}, " - f"but found only {len(gpus)} candidates." - ) - return gpus - - -def _torchrun_argv(script: str, port: int) -> list[str]: - return [ - sys.executable, - "-m", - "torch.distributed.run", - "--standalone", - "--nproc_per_node", - str(_cfg_parallel_size()), - script, - "--port", - str(port), - "--model_path", - str(CHECKPOINT_DIR), - ] - - -def _run_upstream_service(port: int, log_path: Path) -> subprocess.Popen[str]: - env = os.environ.copy() - env.setdefault("PYTHONPATH", "") - env["PYTHONPATH"] = f"{Path.cwd()}:{DREAMZERO_REPO}:{env['PYTHONPATH']}".rstrip(":") - env["CUDA_VISIBLE_DEVICES"] = ",".join(_pick_test_gpus()) - env.setdefault("NO_ALBUMENTATIONS_UPDATE", "1") - env["ATTENTION_BACKEND"] = "torch" - env.setdefault("ENABLE_TENSORRT", "false") - env["ENABLE_DIT_CACHE"] = "false" - env["NUM_DIT_STEPS"] = "16" - env["DYNAMIC_CACHE_SCHEDULE"] = "false" - argv = _torchrun_argv( - str(Path("tests/dreamzero/upstream/upstream_socket_server_no_compile.py")), - port, - ) - log_file = log_path.open("w") - proc = subprocess.Popen( - argv, - stdout=log_file, - stderr=subprocess.STDOUT, - text=True, - cwd=str(Path.cwd()), - env=env, - ) - proc._codex_log_file = log_file # type: ignore[attr-defined] - return proc - - -def _run_vllm_service(port: int, log_path: Path) -> subprocess.Popen[str]: - env = os.environ.copy() - gpus = _pick_test_gpus() - cfg_parallel_size = _cfg_parallel_size() - if cfg_parallel_size > len(gpus): - raise RuntimeError( - f"cfg_parallel_size={cfg_parallel_size} requires at least {cfg_parallel_size} GPUs, but only got {gpus}." - ) - env["CUDA_VISIBLE_DEVICES"] = ",".join(gpus[:cfg_parallel_size]) - env.setdefault("ATTENTION_BACKEND", "torch") - env.setdefault("DIFFUSION_ATTENTION_BACKEND", "TORCH_SDPA") - env.setdefault("MASTER_PORT", str(get_open_port())) - argv = [ - _vllm_executable(), - "serve", - VLLM_MODEL, - "--omni", - "--host", - "127.0.0.1", - "--port", - str(port), - "--served-model-name", - "dreamzero-droid", - "--enforce-eager", - ] - if cfg_parallel_size > 1: - argv.extend(["--cfg-parallel-size", str(cfg_parallel_size)]) - log_file = log_path.open("w") - proc = subprocess.Popen( - argv, - stdout=log_file, - stderr=subprocess.STDOUT, - text=True, - env=env, - cwd=str(Path.cwd()), - ) - proc._codex_log_file = log_file # type: ignore[attr-defined] - return proc - - -def _stop_process(proc: subprocess.Popen[str]) -> None: - log_file = getattr(proc, "_codex_log_file", None) - if proc.poll() is None: - proc.terminate() - try: - proc.wait(timeout=30) - except subprocess.TimeoutExpired: # pragma: no cover - cleanup path - proc.kill() - proc.wait(timeout=10) - if log_file is not None: - log_file.close() - - -def _build_obs_sequence() -> tuple[dict, dict]: - camera_frames = dreamzero_client.load_camera_frames() - chunks = dreamzero_client.build_frame_schedule( - min(v.shape[0] for v in camera_frames.values()), - 1, - ) - obs0 = dreamzero_client._make_obs_from_video(camera_frames, [0], PROMPT, SESSION_ID) - obs1 = dreamzero_client._make_obs_from_video(camera_frames, chunks[0], PROMPT, SESSION_ID) - return obs0, obs1 - - -def _wait_for_client_ready(client_factory, timeout_s: float, proc=None, log_path: Path | None = None): - deadline = time.time() + timeout_s - last_err: Exception | None = None - while time.time() < deadline: - if proc is not None and proc.poll() is not None: - details = "" - if log_path is not None and log_path.exists(): - details = log_path.read_text(errors="replace")[-8000:] - raise RuntimeError(f"Service exited before becoming ready with code {proc.returncode}.\n{details}") - try: - return client_factory() - except Exception as exc: # pragma: no cover - retry path - last_err = exc - time.sleep(1) - raise TimeoutError(f"Timed out waiting for websocket service: {last_err}") - - -def _collect_outputs_with_client(client) -> tuple[dict, list[np.ndarray]]: - metadata = client.get_server_metadata() - obs0, obs1 = _build_obs_sequence() - outputs = [ - client.infer(dict(obs0)), - client.infer(dict(obs1)), - ] - assert _normalize_reset_response(client.reset({})) == "reset successful" - outputs.append(client.infer(dict(obs0))) - client._ws.close() - return metadata, outputs - - -def _normalize_reset_response(response) -> str: - if isinstance(response, str): - return response - decoded = msgpack_numpy.unpackb(response) - if isinstance(decoded, dict): - return str(decoded.get("status")) - return str(decoded) - - -def _normalize_metadata(metadata: dict) -> dict: - normalized = dict(metadata) - if isinstance(normalized.get("image_resolution"), tuple): - normalized["image_resolution"] = list(normalized["image_resolution"]) - return normalized - - -def _assert_logs_clean(log_path: Path) -> None: - text = log_path.read_text(errors="replace") - if "SignalException: Process" in text and "got signal: 15" in text: - text = text.split("Traceback (most recent call last):", 1)[0] - assert "Traceback" not in text, text - assert "RuntimeError:" not in text, text - - -def _assert_upstream_log_matches_vllm_baseline(log_path: Path) -> None: - text = log_path.read_text(errors="replace") - assert "DIT Compute Steps 8 steps" not in text, text - assert "DIT Compute Steps 16 steps" in text, text - - -def test_openpi_service_matches_upstream_server_noncompile(tmp_path: Path) -> None: - expected_metadata = { - "image_resolution": [180, 320], - "n_external_cameras": 2, - "needs_wrist_camera": True, - "needs_stereo_camera": False, - "needs_session_id": True, - "action_space": "joint_position", - } - - upstream_port = get_open_port() - upstream_log = tmp_path / "dreamzero_upstream.log" - upstream_proc = _run_upstream_service(upstream_port, upstream_log) - try: - upstream_client = _wait_for_client_ready( - lambda: WebsocketClientPolicy(host="127.0.0.1", port=upstream_port), - timeout_s=SERVICE_READY_TIMEOUT_S, - proc=upstream_proc, - log_path=upstream_log, - ) - upstream_metadata, upstream_outputs = _collect_outputs_with_client(upstream_client) - finally: - _stop_process(upstream_proc) - _assert_logs_clean(upstream_log) - _assert_upstream_log_matches_vllm_baseline(upstream_log) - - vllm_port = get_open_port() - vllm_log = tmp_path / "vllm_openpi.log" - vllm_proc = _run_vllm_service(vllm_port, vllm_log) - try: - vllm_client = _wait_for_client_ready( - lambda: OpenPIWebsocketClientPolicy(host="127.0.0.1", port=vllm_port), - timeout_s=SERVICE_READY_TIMEOUT_S, - proc=vllm_proc, - log_path=vllm_log, - ) - vllm_metadata, vllm_outputs = _collect_outputs_with_client(vllm_client) - finally: - _stop_process(vllm_proc) - _assert_logs_clean(vllm_log) - - assert _normalize_metadata(upstream_metadata) == expected_metadata - assert _normalize_metadata(vllm_metadata) == expected_metadata - - for idx, (actual, expected) in enumerate(zip(vllm_outputs, upstream_outputs, strict=True)): - np.testing.assert_allclose( - actual, - expected, - rtol=1e-2, - atol=1e-3, - err_msg=f"OpenPI step {idx} output mismatch", - ) diff --git a/tests/e2e/accuracy/conftest.py b/tests/e2e/accuracy/conftest.py index 6c0258aa2..750a81095 100644 --- a/tests/e2e/accuracy/conftest.py +++ b/tests/e2e/accuracy/conftest.py @@ -1,54 +1,15 @@ from __future__ import annotations -import os -import subprocess -from contextlib import contextmanager -from dataclasses import dataclass from io import BytesIO from pathlib import Path import pytest import requests -import torch from PIL import Image -from tests.helpers.runtime import OmniServer, OmniServerParams - def pytest_addoption(parser): group = parser.getgroup("accuracy-e2e") - group.addoption("--gebench-root", action="store", default=None, help="Local GEBench dataset root") - group.addoption("--gedit-root", action="store", default=None, help="Local GEdit-Bench dataset root") - group.addoption( - "--gebench-model", action="store", default="Qwen/Qwen-Image-2512", help="Generate model for GEBench smoke" - ) - group.addoption( - "--gedit-model", action="store", default="Qwen/Qwen-Image-Edit", help="Generate model for GEdit-Bench smoke" - ) - group.addoption( - "--accuracy-judge-model", - action="store", - default="QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ", - help="Judge model path", - ) - group.addoption("--accuracy-gpu", action="store", default="0", help="Single GPU id used sequentially") - group.addoption("--gebench-port", action="store", type=int, default=8093, help="Generate port for GEBench") - group.addoption("--gedit-port", action="store", type=int, default=8093, help="Generate port for GEdit-Bench") - group.addoption( - "--gebench-samples-per-type", - action="store", - type=int, - default=10, - help="Balanced sample count per GEBench type", - ) - group.addoption( - "--gedit-samples-per-group", - action="store", - type=int, - default=20, - help="Balanced sample count per GEdit task group", - ) - group.addoption("--accuracy-workers", action="store", type=int, default=1, help="Worker count for accuracy benches") group.addoption( "--wan22-i2v-image-source", action="store", @@ -77,99 +38,6 @@ def pytest_addoption(parser): ) -def _hf_cache_root() -> Path: - return Path(os.environ.get("HF_HOME", "/root/.cache/huggingface")) - - -def _dataset_cache_dirs(dataset_id: str) -> list[Path]: - cache_root = _hf_cache_root() / "hub" / f"datasets--{dataset_id.replace('/', '--')}" / "snapshots" - if not cache_root.exists(): - return [] - return sorted( - (path for path in cache_root.iterdir() if path.is_dir()), key=lambda path: path.stat().st_mtime, reverse=True - ) - - -def _ensure_dataset_snapshot(dataset_id: str) -> Path: - candidates = _dataset_cache_dirs(dataset_id) - if candidates: - return candidates[0] - - subprocess.run( - ["hf", "download", "--repo-type", "dataset", dataset_id], - check=True, - ) - candidates = _dataset_cache_dirs(dataset_id) - if not candidates: - raise FileNotFoundError( - f"Dataset {dataset_id} was downloaded but no snapshot was found under {_hf_cache_root()}" - ) - return candidates[0] - - -def _resolve_dataset_root(request: pytest.FixtureRequest, option_name: str, dataset_id: str) -> Path: - value = request.config.getoption(option_name) - if value: - path = Path(value) - if not path.exists(): - pytest.skip(f"Dataset path does not exist: {path}") - return path - return _ensure_dataset_snapshot(dataset_id) - - -@dataclass -class AccuracyServerConfig: - generate_params: OmniServerParams - judge_params: OmniServerParams - run_level: str - model_prefix: str - - @contextmanager - def generate_server(self): - params = self.generate_params - model = self.model_prefix + params.model - server_args = params.server_args or [] - if params.use_omni and params.stage_init_timeout is not None: - server_args = ["--stage-init-timeout", str(params.stage_init_timeout), *server_args] - with OmniServer( - model, - server_args, - port=params.port, - env_dict=params.env_dict, - use_omni=params.use_omni, - ) as server: - yield server - - @contextmanager - def judge_server(self): - params = self.judge_params - model = self.model_prefix + params.model - server_args = params.server_args or [] - with OmniServer( - model, - server_args, - port=params.port, - env_dict=params.env_dict, - use_omni=params.use_omni, - ) as server: - yield server - - -@pytest.fixture(scope="session") -def gebench_dataset_root(request: pytest.FixtureRequest) -> Path: - return _resolve_dataset_root(request, "gebench_root", "stepfun-ai/GEBench") - - -@pytest.fixture(scope="session") -def gedit_dataset_root(request: pytest.FixtureRequest) -> Path: - return _resolve_dataset_root(request, "gedit_root", "stepfun-ai/GEdit-Bench") - - -@pytest.fixture(scope="session") -def accuracy_workers(request: pytest.FixtureRequest) -> int: - return int(request.config.getoption("accuracy_workers")) - - @pytest.fixture(scope="session") def wan22_i2v_image_source(request: pytest.FixtureRequest) -> str | None: value = request.config.getoption("wan22_i2v_image_source") @@ -192,16 +60,6 @@ def hunyuanvideo15_online_timeout_seconds(request: pytest.FixtureRequest) -> int return int(request.config.getoption("hunyuanvideo15_online_timeout_seconds")) -@pytest.fixture(scope="session") -def gebench_samples_per_type(request: pytest.FixtureRequest) -> int: - return int(request.config.getoption("gebench_samples_per_type")) - - -@pytest.fixture(scope="session") -def gedit_samples_per_group(request: pytest.FixtureRequest) -> int: - return int(request.config.getoption("gedit_samples_per_group")) - - @pytest.fixture(scope="session") def accuracy_artifact_root() -> Path: root = Path(__file__).resolve().parent / "artifacts" @@ -209,12 +67,6 @@ def accuracy_artifact_root() -> Path: return root -@pytest.fixture(scope="session") -def accuracy_assets_root() -> Path: - root = Path(__file__).resolve().parent / "assets" - return root - - @pytest.fixture(scope="session") def qwen_bear_image(accuracy_artifact_root: Path): """Download the Qwen bear image from the URL and save it to the accuracy artifact root.""" @@ -249,80 +101,3 @@ def rabbit_image(accuracy_artifact_root: Path): image.save(image_path) yield image image.close() - - -def _build_accuracy_server_config( - *, - generate_model: str, - judge_model: str, - shared_gpu: str, - port: int, - run_level: str, - model_prefix: str, -) -> AccuracyServerConfig: - if torch.accelerator.device_count() < 1: - pytest.skip("Need at least 1 CUDA GPU for accuracy benchmark smoke tests.") - - if not generate_model: - pytest.skip("No generate model configured for accuracy benchmark test.") - generate_server_args = ["--num-gpus", "1"] - judge_server_args = [ - "--max-model-len", - "32768", - "--gpu-memory-utilization", - "0.8", - ] - - judge_env = {"CUDA_VISIBLE_DEVICES": shared_gpu} - - return AccuracyServerConfig( - generate_params=OmniServerParams( - model=generate_model, - port=port, - server_args=generate_server_args, - env_dict={"CUDA_VISIBLE_DEVICES": shared_gpu}, - use_omni=True, - stage_init_timeout=300, - ), - judge_params=OmniServerParams( - model=judge_model, - port=port, - server_args=judge_server_args, - env_dict=judge_env, - use_omni=False, - ), - run_level=run_level, - model_prefix=model_prefix, - ) - - -@pytest.fixture -def gebench_accuracy_servers( - request: pytest.FixtureRequest, - run_level: str, - model_prefix: str, -) -> AccuracyServerConfig: - return _build_accuracy_server_config( - generate_model=request.config.getoption("gebench_model"), - judge_model=request.config.getoption("accuracy_judge_model"), - shared_gpu=str(request.config.getoption("accuracy_gpu")), - port=int(request.config.getoption("gebench_port")), - run_level=run_level, - model_prefix=model_prefix, - ) - - -@pytest.fixture -def gedit_accuracy_servers( - request: pytest.FixtureRequest, - run_level: str, - model_prefix: str, -) -> AccuracyServerConfig: - return _build_accuracy_server_config( - generate_model=request.config.getoption("gedit_model"), - judge_model=request.config.getoption("accuracy_judge_model"), - shared_gpu=str(request.config.getoption("accuracy_gpu")), - port=int(request.config.getoption("gedit_port")), - run_level=run_level, - model_prefix=model_prefix, - ) diff --git a/tests/e2e/accuracy/helpers.py b/tests/e2e/accuracy/helpers.py index ebe635ea8..f3816136c 100644 --- a/tests/e2e/accuracy/helpers.py +++ b/tests/e2e/accuracy/helpers.py @@ -3,8 +3,12 @@ import os import re import shutil import subprocess +import sys import time +from abc import ABC, abstractmethod from base64 import b64decode, b64encode +from collections.abc import Callable, Iterator +from contextlib import contextmanager from fractions import Fraction from hashlib import sha1 from io import BytesIO @@ -53,8 +57,6 @@ def env_to_apply_ftfy_mock_in_subproc(env: dict[str, str] | None = None) -> dict def reset_artifact_dir(path: Path) -> Path: - import shutil - if path.exists(): shutil.rmtree(path) path.mkdir(parents=True, exist_ok=True) @@ -609,3 +611,344 @@ def download_images(urls: list[str]) -> list[Image.Image]: images.append(image) return images + + +_DREAMZERO_UPSTREAM_SERVER_SCRIPT = ( + Path(__file__).resolve().parent / "upstream_mock" / "upstream_socket_server_no_compile.py" +) +_DREAMZERO_DEFAULT_MODEL = "GEAR-Dreams/DreamZero-DROID" +_DREAMZERO_DEFAULT_READY_TIMEOUT_S = int(os.environ.get("OPENPI_SERVICE_READY_TIMEOUT_S", "900")) +_DREAMZERO_DEFAULT_PROMPT = ( + "Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan" +) +_DREAMZERO_DEFAULT_SESSION_ID = "openpi-e2e-parity-session" +_DREAMZERO_OPENPI_VLLM_PATH = "/v1/realtime/robot/openpi" + + +def require_dreamzero_parity_env() -> tuple[Path, Path]: + repo_env = os.environ.get("DREAMZERO_REPO") + if repo_env is None: + raise ValueError("DREAMZERO_REPO environment variable is required.") + repo = Path(repo_env).expanduser() + if not repo.is_dir(): + raise FileNotFoundError(f"DREAMZERO_REPO does not exist: {repo}") + checkpoint_dir = repo / "checkpoints" / "dreamzero" + if not checkpoint_dir.is_dir(): + raise FileNotFoundError(f"DreamZero checkpoint not found: {checkpoint_dir}") + if str(repo) not in sys.path: + sys.path.insert(0, str(repo)) + return repo, checkpoint_dir + + +def load_dreamzero_upstream_modules(): + require_dreamzero_parity_env() + import test_client_AR as dreamzero_upstream_client + from eval_utils.policy_client import WebsocketClientPolicy + + return dreamzero_upstream_client, WebsocketClientPolicy + + +def dreamzero_cfg_parallel_size() -> int: + return int(os.environ.get("OPENPI_E2E_CFG_PARALLEL_SIZE", "2")) + + +def _dreamzero_parity_vllm_env() -> dict[str, str]: + from tests.helpers.env import pick_least_used_device_indices + from tests.helpers.runtime import get_open_port + + parallel_size = dreamzero_cfg_parallel_size() + gpus = [str(index) for index in pick_least_used_device_indices(parallel_size)] + if parallel_size > len(gpus): + raise RuntimeError( + f"cfg_parallel_size={parallel_size} requires at least {parallel_size} GPUs, but only got {gpus}." + ) + return { + "CUDA_VISIBLE_DEVICES": ",".join(gpus[:parallel_size]), + "ATTENTION_BACKEND": "torch", + "DIFFUSION_ATTENTION_BACKEND": "TORCH_SDPA", + "VLLM_DISABLE_COMPILE_CACHE": "1", + "MASTER_PORT": str(get_open_port()), + } + + +def dreamzero_parity_vllm_params(model: str | None = None): + from tests.helpers.runtime import OmniServerParams + from tests.helpers.stage_config import get_deploy_config_path + + parallel_size = dreamzero_cfg_parallel_size() + server_args = [ + "--served-model-name", + "dreamzero-droid", + "--enforce-eager", + ] + if parallel_size > 1: + server_args.extend(["--cfg-parallel-size", str(parallel_size)]) + return OmniServerParams( + model=model or os.environ.get("VLLM_DREAMZERO_MODEL", _DREAMZERO_DEFAULT_MODEL), + stage_config_path=get_deploy_config_path("dreamzero_tp1_cfg2.yaml"), + server_args=server_args, + env_dict=_dreamzero_parity_vllm_env(), + ) + + +def _dreamzero_omni_server_from_params(params): + from tests.helpers.runtime import OmniServer + + server_args = list(params.server_args or []) + if params.stage_config_path is not None: + server_args += ["--stage-configs-path", params.stage_config_path] + return OmniServer( + params.model, + server_args, + port=params.port, + env_dict=params.env_dict, + use_omni=params.use_omni, + ) + + +@contextmanager +def dreamzero_parity_vllm_client( + params=None, + *, + ready_timeout_s: float = _DREAMZERO_DEFAULT_READY_TIMEOUT_S, +) -> Iterator["DreamZeroParityClient"]: + """Start vLLM via ``OmniServer`` and return a ready OpenPI parity client.""" + with _dreamzero_omni_server_from_params(params or dreamzero_parity_vllm_params()) as server: + client = _dreamzero_wait_for_client_ready( + lambda: _dreamzero_create_openpi_vllm_client(server.host, server.port), + timeout_s=ready_timeout_s, + proc=server.proc, + ) + yield DreamZeroParityClient(client) + + +def _dreamzero_create_openpi_vllm_client(host: str, port: int) -> Any: + from openpi_client import msgpack_numpy + + _, WebsocketClientPolicy = load_dreamzero_upstream_modules() + + class _OpenPIWebsocketClientPolicy(WebsocketClientPolicy): + def __init__( + self, + host: str = "127.0.0.1", + port: int = 8000, + path: str = _DREAMZERO_OPENPI_VLLM_PATH, + ) -> None: + self._uri = f"ws://{host}:{port}{path}" + self._packer = msgpack_numpy.Packer() + self._ws, self._server_metadata = self._wait_for_server() + + return _OpenPIWebsocketClientPolicy(host=host, port=port) + + +def _dreamzero_stop_process(proc: subprocess.Popen[str]) -> None: + log_file = getattr(proc, "_codex_log_file", None) + if proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=30) + except subprocess.TimeoutExpired: # pragma: no cover - cleanup path + proc.kill() + proc.wait(timeout=10) + if log_file is not None: + log_file.close() + + +def _dreamzero_wait_for_client_ready( + client_factory: Callable[[], Any], + *, + timeout_s: float, + proc: subprocess.Popen[str] | None = None, + log_path: Path | None = None, +) -> Any: + deadline = time.time() + timeout_s + last_err: Exception | None = None + while time.time() < deadline: + if proc is not None and proc.poll() is not None: + details = "" + if log_path is not None and log_path.exists(): + details = log_path.read_text(errors="replace")[-8000:] + raise RuntimeError(f"Service exited before becoming ready with code {proc.returncode}.\n{details}") + try: + return client_factory() + except Exception as exc: # pragma: no cover - retry path + last_err = exc + time.sleep(1) + raise TimeoutError(f"Timed out waiting for websocket service: {last_err}") + + +def _dreamzero_normalize_reset_response(response: Any) -> str: + from openpi_client import msgpack_numpy + + if isinstance(response, str): + return response + decoded = msgpack_numpy.unpackb(response) + if isinstance(decoded, dict): + return str(decoded.get("status")) + return str(decoded) + + +def normalize_dreamzero_metadata(metadata: dict[str, Any]) -> dict[str, Any]: + normalized = dict(metadata) + if isinstance(normalized.get("image_resolution"), tuple): + normalized["image_resolution"] = list(normalized["image_resolution"]) + return normalized + + +def assert_dreamzero_logs_clean(log_path: Path) -> None: + text = log_path.read_text(errors="replace") + if "SignalException: Process" in text and "got signal: 15" in text: + text = text.split("Traceback (most recent call last):", 1)[0] + assert "Traceback" not in text, text + assert "RuntimeError:" not in text, text + + +def assert_dreamzero_upstream_log_matches_vllm_baseline(log_path: Path) -> None: + text = log_path.read_text(errors="replace") + assert "DIT Compute Steps 8 steps" not in text, text + assert "DIT Compute Steps 16 steps" in text, text + + +class DreamZeroParityClient: + """Connected OpenPI websocket client for DreamZero parity inference.""" + + def __init__(self, client: Any) -> None: + self._client = client + + @property + def raw_client(self) -> Any: + return self._client + + def build_obs_sequence(self) -> tuple[dict[str, Any], dict[str, Any]]: + dreamzero_upstream_client, _ = load_dreamzero_upstream_modules() + camera_frames = dreamzero_upstream_client.load_camera_frames() + chunks = dreamzero_upstream_client.build_frame_schedule( + min(v.shape[0] for v in camera_frames.values()), + 1, + ) + obs0 = dreamzero_upstream_client._make_obs_from_video( + camera_frames, + [0], + _DREAMZERO_DEFAULT_PROMPT, + _DREAMZERO_DEFAULT_SESSION_ID, + ) + obs1 = dreamzero_upstream_client._make_obs_from_video( + camera_frames, + chunks[0], + _DREAMZERO_DEFAULT_PROMPT, + _DREAMZERO_DEFAULT_SESSION_ID, + ) + return obs0, obs1 + + def collect_parity_outputs(self) -> tuple[dict[str, Any], list[np.ndarray]]: + metadata = self._client.get_server_metadata() + obs0, obs1 = self.build_obs_sequence() + outputs = [ + self._client.infer(dict(obs0)), + self._client.infer(dict(obs1)), + ] + assert _dreamzero_normalize_reset_response(self._client.reset({})) == "reset successful" + outputs.append(self._client.infer(dict(obs0))) + self._client._ws.close() + return metadata, outputs + + +class _DreamZeroParityServer(ABC): + host = "127.0.0.1" + + def __init__( + self, + *, + log_path: Path, + port: int | None = None, + ready_timeout_s: float = _DREAMZERO_DEFAULT_READY_TIMEOUT_S, + ) -> None: + from tests.helpers.runtime import get_open_port + + self.log_path = log_path + self.port = get_open_port() if port is None else port + self.ready_timeout_s = ready_timeout_s + self.proc: subprocess.Popen[str] | None = None + self._log_file = None + + @abstractmethod + def _build_argv(self) -> list[str]: + raise NotImplementedError + + @abstractmethod + def _build_env(self) -> dict[str, str]: + raise NotImplementedError + + @abstractmethod + def _create_client(self) -> Any: + raise NotImplementedError + + def _start(self) -> None: + self.log_path.parent.mkdir(parents=True, exist_ok=True) + self._log_file = self.log_path.open("w") + self.proc = subprocess.Popen( + self._build_argv(), + stdout=self._log_file, + stderr=subprocess.STDOUT, + text=True, + cwd=str(Path.cwd()), + env=self._build_env(), + ) + self.proc._codex_log_file = self._log_file # type: ignore[attr-defined] + + def __enter__(self) -> DreamZeroParityClient: + self._start() + client = _dreamzero_wait_for_client_ready( + self._create_client, + timeout_s=self.ready_timeout_s, + proc=self.proc, + log_path=self.log_path, + ) + return DreamZeroParityClient(client) + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + if self.proc is not None: + _dreamzero_stop_process(self.proc) + self.proc = None + + +class DreamZeroUpstreamServer(_DreamZeroParityServer): + """Official DreamZero upstream reference server launched via torchrun.""" + + def _build_argv(self) -> list[str]: + _, checkpoint_dir = require_dreamzero_parity_env() + return [ + sys.executable, + "-m", + "torch.distributed.run", + "--standalone", + "--nproc_per_node", + str(dreamzero_cfg_parallel_size()), + str(_DREAMZERO_UPSTREAM_SERVER_SCRIPT), + "--port", + str(self.port), + "--model_path", + str(checkpoint_dir), + ] + + def _build_env(self) -> dict[str, str]: + from tests.helpers.env import pick_least_used_device_indices + + dreamzero_repo, _ = require_dreamzero_parity_env() + env = os.environ.copy() + env.setdefault("PYTHONPATH", "") + env["PYTHONPATH"] = f"{Path.cwd()}:{dreamzero_repo}:{env['PYTHONPATH']}".rstrip(":") + env["CUDA_VISIBLE_DEVICES"] = ",".join( + str(index) for index in pick_least_used_device_indices(max(dreamzero_cfg_parallel_size(), 1)) + ) + env.setdefault("NO_ALBUMENTATIONS_UPDATE", "1") + env["ATTENTION_BACKEND"] = "torch" + env.setdefault("ENABLE_TENSORRT", "false") + env["ENABLE_DIT_CACHE"] = "false" + env["NUM_DIT_STEPS"] = "16" + env["DYNAMIC_CACHE_SCHEDULE"] = "false" + return env + + def _create_client(self) -> Any: + _, WebsocketClientPolicy = load_dreamzero_upstream_modules() + return WebsocketClientPolicy(host=self.host, port=self.port) diff --git a/tests/e2e/accuracy/test_dreamzero.py b/tests/e2e/accuracy/test_dreamzero.py new file mode 100644 index 000000000..2507530b3 --- /dev/null +++ b/tests/e2e/accuracy/test_dreamzero.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""DreamZero OpenPI upstream parity accuracy test.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import numpy as np +import pytest + +from tests.e2e.accuracy.helpers import ( + DreamZeroUpstreamServer, + assert_dreamzero_logs_clean, + assert_dreamzero_upstream_log_matches_vllm_baseline, + dreamzero_parity_vllm_client, + load_dreamzero_upstream_modules, + normalize_dreamzero_metadata, +) +from tests.helpers.mark import hardware_test + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" +os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" + +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] + +EXPECTED_METADATA = { + "image_resolution": [180, 320], + "n_external_cameras": 2, + "needs_wrist_camera": True, + "needs_stereo_camera": False, + "needs_session_id": True, + "action_space": "joint_position", +} + + +@hardware_test(res={"cuda": "H100"}, num_cards=2) +def test_dreamzero_openpi_upstream_parity(tmp_path: Path) -> None: + """Assert vLLM OpenPI actions match the upstream DreamZero reference server.""" + load_dreamzero_upstream_modules() + + upstream_log = tmp_path / "dreamzero_upstream.log" + with DreamZeroUpstreamServer(log_path=upstream_log) as upstream_client: + upstream_metadata, upstream_outputs = upstream_client.collect_parity_outputs() + assert_dreamzero_logs_clean(upstream_log) + assert_dreamzero_upstream_log_matches_vllm_baseline(upstream_log) + + with dreamzero_parity_vllm_client() as vllm_client: + vllm_metadata, vllm_outputs = vllm_client.collect_parity_outputs() + + assert normalize_dreamzero_metadata(upstream_metadata) == EXPECTED_METADATA + assert normalize_dreamzero_metadata(vllm_metadata) == EXPECTED_METADATA + + for idx, (vllm_action, upstream_action) in enumerate(zip(vllm_outputs, upstream_outputs, strict=True)): + np.testing.assert_allclose( + vllm_action, + upstream_action, + rtol=1e-2, + atol=1e-3, + err_msg=f"OpenPI step {idx} output mismatch", + ) diff --git a/tests/e2e/accuracy/test_gebench_h100_smoke.py b/tests/e2e/accuracy/test_gebench_h100_smoke.py deleted file mode 100644 index b17c951e5..000000000 --- a/tests/e2e/accuracy/test_gebench_h100_smoke.py +++ /dev/null @@ -1,97 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from benchmarks.accuracy.text_to_image.gbench import main as gbench_main -from tests.e2e.accuracy.helpers import infer_model_label, reset_artifact_dir -from tests.helpers.mark import hardware_test - -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] - - -@pytest.mark.benchmark -@hardware_test(res={"cuda": "H100"}, num_cards=1) -@pytest.mark.skip(reason="#3256") -def test_gebench_h100_smoke( - gebench_accuracy_servers, - accuracy_artifact_root: Path, - gebench_dataset_root: Path, - accuracy_workers: int, -) -> None: - model_label = infer_model_label(gebench_accuracy_servers.generate_params.model).lower() - output_root = reset_artifact_dir(accuracy_artifact_root / f"gebench_{model_label}") - - with gebench_accuracy_servers.generate_server() as generate_server: - for data_type in ("type3", "type4"): - assert ( - gbench_main( - [ - "generate", - "--dataset-root", - str(gebench_dataset_root), - "--output-root", - str(output_root), - "--base-url", - f"http://{generate_server.host}:{generate_server.port}", - "--model", - generate_server.model, - "--data-type", - data_type, - "--width", - "768", - "--height", - "576", - "--output-compression", - "98", - "--num-inference-steps", - "8", - "--workers", - str(accuracy_workers), - ] - ) - == 0 - ) - - with gebench_accuracy_servers.judge_server() as judge_server: - for data_type in ("type3", "type4"): - assert ( - gbench_main( - [ - "evaluate", - "--dataset-root", - str(gebench_dataset_root), - "--output-root", - str(output_root), - "--data-type", - data_type, - "--judge-base-url", - f"http://{judge_server.host}:{judge_server.port}", - "--judge-model", - judge_server.model, - "--judge-api-key", - "EMPTY", - "--workers", - str(accuracy_workers), - ] - ) - == 0 - ) - - assert gbench_main(["summarize", "--output-root", str(output_root)]) == 0 - - summary = json.loads((output_root / "summary.json").read_text(encoding="utf-8")) - assert "generation" in summary - assert "evaluation" in summary - - for data_type in ("type3", "type4"): - assert data_type in summary["generation"]["by_type"] - assert summary["generation"]["by_type"][data_type]["count"] > 0 - assert data_type in summary["evaluation"]["by_type"] - assert summary["evaluation"]["by_type"][data_type]["count"] > 0 - - assert summary["evaluation"]["overall_mean"] >= 0.45 - assert summary["evaluation"]["by_type"]["type3"]["overall_mean"] >= 0.45 - assert summary["evaluation"]["by_type"]["type4"]["overall_mean"] >= 0.45 diff --git a/tests/e2e/accuracy/test_gedit_bench_h100_smoke.py b/tests/e2e/accuracy/test_gedit_bench_h100_smoke.py deleted file mode 100644 index a7831fac8..000000000 --- a/tests/e2e/accuracy/test_gedit_bench_h100_smoke.py +++ /dev/null @@ -1,115 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from benchmarks.accuracy.image_to_image.gedit_bench import GROUPS -from benchmarks.accuracy.image_to_image.gedit_bench import main as gedit_main -from tests.e2e.accuracy.helpers import infer_model_label, reset_artifact_dir -from tests.helpers.mark import hardware_test - -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] - - -@pytest.mark.benchmark -@hardware_test(res={"cuda": "H100"}, num_cards=1) -@pytest.mark.skip(reason="#3257") -def test_gedit_bench_h100_smoke( - gedit_accuracy_servers, - accuracy_artifact_root: Path, - gedit_dataset_root: Path, - gedit_samples_per_group: int, - accuracy_workers: int, -) -> None: - model_label = infer_model_label(gedit_accuracy_servers.generate_params.model).lower() - output_root = reset_artifact_dir(accuracy_artifact_root / f"gedit_results_{model_label}") - score_root = reset_artifact_dir(accuracy_artifact_root / f"gedit_scores_{model_label}") - model_name = model_label - - with gedit_accuracy_servers.generate_server() as generate_server: - assert ( - gedit_main( - [ - "generate", - "--dataset-ref", - str(gedit_dataset_root), - "--output-root", - str(output_root), - "--base-url", - f"http://{generate_server.host}:{generate_server.port}", - "--model", - generate_server.model, - "--model-name", - model_name, - "--task-type", - "all", - "--instruction-language", - "all", - "--samples-per-group", - str(gedit_samples_per_group), - "--workers", - str(accuracy_workers), - ] - ) - == 0 - ) - - with gedit_accuracy_servers.judge_server() as judge_server: - assert ( - gedit_main( - [ - "evaluate", - "--dataset-ref", - str(gedit_dataset_root), - "--output-root", - str(output_root), - "--model-name", - model_name, - "--save-dir", - str(score_root), - "--task-type", - "all", - "--instruction-language", - "all", - "--judge-base-url", - f"http://{judge_server.host}:{judge_server.port}", - "--judge-model", - judge_server.model, - "--judge-api-key", - "EMPTY", - "--samples-per-group", - str(gedit_samples_per_group), - "--workers", - str(accuracy_workers), - ] - ) - == 0 - ) - - csv_path = score_root / f"{model_name}_all_all_vie_score.csv" - assert gedit_main(["summarize", "--csv-path", str(csv_path), "--language", "all"]) == 0 - - summary_path = score_root / f"{model_name}_all_all_summary.json" - summary = json.loads(summary_path.read_text(encoding="utf-8")) - assert set(summary["languages"]) == {"en", "cn"} - - for language in ["en", "cn"]: - language_summary = summary["languages"][language] - assert language_summary["overall"]["count"] is not None - assert language_summary["intersection"]["count"] is not None - assert language_summary["overall"]["Q_SC"] is not None - assert language_summary["overall"]["Q_PQ"] is not None - assert language_summary["overall"]["Q_O"] is not None - - for group in GROUPS: - group_summary = language_summary["by_group"][group] - assert set(group_summary) == {"count", "Q_SC", "Q_PQ", "Q_O"} - - assert summary["languages"]["en"]["overall"]["Q_SC"] >= 6.95 - assert summary["languages"]["en"]["overall"]["Q_PQ"] >= 5.8 - assert summary["languages"]["en"]["overall"]["Q_O"] >= 6.15 - assert summary["languages"]["cn"]["overall"]["Q_SC"] >= 6.9 - assert summary["languages"]["cn"]["overall"]["Q_PQ"] >= 5.7 - assert summary["languages"]["cn"]["overall"]["Q_O"] >= 6.1 diff --git a/tests/e2e/accuracy/test_hunyuan_image3.py b/tests/e2e/accuracy/test_hunyuan_image3.py index e2d84b13d..8c4570fe3 100644 --- a/tests/e2e/accuracy/test_hunyuan_image3.py +++ b/tests/e2e/accuracy/test_hunyuan_image3.py @@ -28,6 +28,7 @@ from tests.e2e.accuracy.helpers import ( model_output_dir, ) from tests.helpers.mark import hardware_test +from tests.helpers.media import get_asset_path from tests.helpers.runtime import OmniRunner, OmniServer from vllm_omni.diffusion.models.hunyuan_image3.prompt_utils import ( build_prompt_tokens, @@ -94,6 +95,7 @@ def _default_ar_dit_devices() -> tuple[str, str]: AR_DEVICES, DIT_DEVICES = _default_ar_dit_devices() MODEL_NAME = "tencent/HunyuanImage-3.0-Instruct" +HUNYUAN_IMAGE_REF_PATH = get_asset_path("hunyuan/hunyuan_image_ref.png") NUM_INFERENCE_STEPS = 50 GUIDANCE_SCALE = 2.5 @@ -414,7 +416,7 @@ def _run_online(stage_configs_path: str, output_path: Path) -> tuple[Image.Image torch.accelerator.device_count() < AR_TP_SIZE + DIT_TP_SIZE, reason=f"Needs {AR_TP_SIZE + DIT_TP_SIZE}+ GPUs ({AR_TP_SIZE} AR + {DIT_TP_SIZE} DiT)", ) -def test_image_to_image_alignment_online(accuracy_artifact_root: Path, accuracy_assets_root: Path) -> None: +def test_image_to_image_alignment_online(accuracy_artifact_root: Path) -> None: """Online API test: same pipeline, same seed as offline → PSNR >= 10 dB.""" if importlib.util.find_spec("FlagEmbedding") is None: raise ImportError("Missing dependency: FlagEmbedding\nInstall with: pip install FlagEmbedding") @@ -431,7 +433,7 @@ def test_image_to_image_alignment_online(accuracy_artifact_root: Path, accuracy_ scorer = SemanticSimilarityScorer() clip_scorer = CLIPScorer() cot_results = scorer.text_similarity(online_cot, COT_REF) - image_ref = Image.open(str(accuracy_assets_root / "hunyuan_image_ref.png")).convert("RGB") + image_ref = Image.open(HUNYUAN_IMAGE_REF_PATH).convert("RGB") image_clip_score = clip_scorer.image_image_score(online_image, image_ref) ssim_value, psnr_value = compute_image_ssim_psnr(prediction=online_image, reference=image_ref, compare_mode="RGB") @@ -529,7 +531,7 @@ def _run_dit_model( torch.accelerator.device_count() < AR_TP_SIZE + DIT_TP_SIZE, reason=f"Needs {AR_TP_SIZE + DIT_TP_SIZE}+ GPUs ({AR_TP_SIZE} AR + {DIT_TP_SIZE} DiT)", ) -def test_image_to_image_alignment(accuracy_artifact_root: Path, accuracy_assets_root: Path) -> None: +def test_image_to_image_alignment(accuracy_artifact_root: Path) -> None: if importlib.util.find_spec("FlagEmbedding") is None: raise ImportError("Missing dependency: FlagEmbedding\nInstall with: pip install FlagEmbedding") from tabulate import tabulate # lazy import @@ -545,7 +547,7 @@ def test_image_to_image_alignment(accuracy_artifact_root: Path, accuracy_assets_ scorer = SemanticSimilarityScorer() clip_scorer = CLIPScorer() cot_results = scorer.text_similarity(omni_cot, COT_REF) - image_ref = Image.open(str(accuracy_assets_root / "hunyuan_image_ref.png")).convert("RGB") + image_ref = Image.open(HUNYUAN_IMAGE_REF_PATH).convert("RGB") image_clip_score = clip_scorer.image_image_score(omni_image, image_ref) ssim_value, psnr_value = compute_image_ssim_psnr(prediction=omni_image, reference=image_ref, compare_mode="RGB") diff --git a/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py b/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py index 197044ef3..34131dc5a 100644 --- a/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py +++ b/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py @@ -15,6 +15,7 @@ from PIL import Image from tests.e2e.accuracy.helpers import assert_images_pixel_close, assert_similarity, model_output_dir from tests.helpers.mark import hardware_test +from tests.helpers.media import get_asset_path from tests.helpers.runtime import OmniServer pytestmark = [pytest.mark.full_model, pytest.mark.diffusion] @@ -33,8 +34,8 @@ P99_THRESHOLD = 3e-1 SSIM_THRESHOLD = 0.97 PSNR_THRESHOLD = 30.0 +BASELINE_PATH = get_asset_path("hunyuan/hunyuan_baseline.png") _REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent -BASELINE_PATH = _REPO_ROOT / "tests" / "assets" / "hunyuan" / "hunyuan_baseline.png" _OFFLINE_SCRIPT = _REPO_ROOT / "examples" / "offline_inference" / "hunyuan_image3" / "end2end.py" # DiT-only deploy config with trust_remote_code (based on hunyuan_image3_dit.yaml). diff --git a/tests/e2e/accuracy/test_qwen_image_edit.py b/tests/e2e/accuracy/test_qwen_image_edit.py index 338758cc0..9ee9c2c87 100644 --- a/tests/e2e/accuracy/test_qwen_image_edit.py +++ b/tests/e2e/accuracy/test_qwen_image_edit.py @@ -14,6 +14,7 @@ from benchmarks.accuracy.common import decode_base64_image, pil_to_png_bytes from tests.e2e.accuracy.helpers import assert_images_pixel_close, assert_similarity, model_output_dir from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup from tests.helpers.mark import hardware_test +from tests.helpers.media import get_asset_path from tests.helpers.runtime import OmniServer pytestmark = [pytest.mark.full_model, pytest.mark.diffusion] @@ -46,10 +47,9 @@ EDIT_2511_PROMPT = ( "（帽子紧贴头顶，未贴合部分可调整帽子摆放角度达到完全贴合）， 头部与身体过渡必须自然，无拼接痕迹，" "无额外元素出现在头部四周，纯白背景，不要出现绿色圆圈" ) -EDIT_2511_ASSET_DIR = Path(__file__).resolve().parent / "assets" EDIT_2511_IMAGE_PATHS = [ - EDIT_2511_ASSET_DIR / "qwen_image_edit_2511_test1.png", - EDIT_2511_ASSET_DIR / "qwen_image_edit_2511_test2.png", + get_asset_path("qwen_image_edit/qwen_image_edit_2511_test1.png"), + get_asset_path("qwen_image_edit/qwen_image_edit_2511_test2.png"), ] PROMPT_SINGLE_IMAGE = "The input is a 2D cartoon bear mascot. Restyle it into a painterly oil artwork with warm colors while preserving the main structure." diff --git a/tests/dreamzero/upstream/upstream_socket_server_no_compile.py b/tests/e2e/accuracy/upstream_mock/upstream_socket_server_no_compile.py similarity index 98% rename from tests/dreamzero/upstream/upstream_socket_server_no_compile.py rename to tests/e2e/accuracy/upstream_mock/upstream_socket_server_no_compile.py index a9ef7a3d0..e6fccdd23 100644 --- a/tests/dreamzero/upstream/upstream_socket_server_no_compile.py +++ b/tests/e2e/accuracy/upstream_mock/upstream_socket_server_no_compile.py @@ -24,7 +24,7 @@ PyTorch SDPA (`ATTENTION_BACKEND=torch`) for this subprocess only. Usage: PYTHONPATH="${DREAMZERO_REPO}" \\ .venv/bin/python -m torch.distributed.run --standalone --nproc_per_node=2 \\ - tests/dreamzero/upstream/upstream_socket_server_no_compile.py --port 18081 \\ + tests/e2e/accuracy/upstream_mock/upstream_socket_server_no_compile.py --port 18081 \\ --model_path "${DREAMZERO_REPO}/checkpoints/dreamzero" """ diff --git a/tests/comfyui/conftest.py b/tests/e2e/features/comfyui/conftest.py similarity index 97% rename from tests/comfyui/conftest.py rename to tests/e2e/features/comfyui/conftest.py index 4280d3506..c3ac99bf2 100644 --- a/tests/comfyui/conftest.py +++ b/tests/e2e/features/comfyui/conftest.py @@ -25,7 +25,7 @@ def _setup_comfyui_test_environment(): """Set up the test environment for ComfyUI plugin testing.""" # === Add ComfyUI plugin path to allow importing comfyui_vllm_omni === _COMFYUI_PLUGIN_PATH = os.path.abspath( - os.path.join(os.path.dirname(__file__), "..", "..", "apps", "ComfyUI-vLLM-Omni") + os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "apps", "ComfyUI-vLLM-Omni") ) if not os.path.isdir(_COMFYUI_PLUGIN_PATH): raise FileNotFoundError( diff --git a/tests/comfyui/test_comfyui_integration.py b/tests/e2e/features/comfyui/test_comfyui_integration.py similarity index 99% rename from tests/comfyui/test_comfyui_integration.py rename to tests/e2e/features/comfyui/test_comfyui_integration.py index e15098b4e..2024cce8b 100644 --- a/tests/comfyui/test_comfyui_integration.py +++ b/tests/e2e/features/comfyui/test_comfyui_integration.py @@ -6,6 +6,8 @@ It ensures that 2. The sampling parameters are correctly passed from the node to AsyncOmni through the API layer. """ +from __future__ import annotations + import multiprocessing import time import traceback @@ -52,7 +54,7 @@ class ServerCase(NamedTuple): class SamplingCase(NamedTuple): """Parametrizing the input sampling parameters.""" - kind: "SamplingKind" + kind: SamplingKind sampling_params: dict | list[dict] | None lora: dict | None = None diff --git a/tests/e2e/offline_inference/custom_pipeline/test_async_omni_collective_rpc.py b/tests/e2e/features/custom_pipeline/test_async_omni_collective_rpc.py similarity index 96% rename from tests/e2e/offline_inference/custom_pipeline/test_async_omni_collective_rpc.py rename to tests/e2e/features/custom_pipeline/test_async_omni_collective_rpc.py index dc701af7a..04e83caeb 100644 --- a/tests/e2e/offline_inference/custom_pipeline/test_async_omni_collective_rpc.py +++ b/tests/e2e/features/custom_pipeline/test_async_omni_collective_rpc.py @@ -16,7 +16,7 @@ exercises when it calls ``await self.engine.list_loras()`` before dispatching a generation request. Usage: - pytest tests/e2e/offline_inference/custom_pipeline/test_async_omni_collective_rpc.py -v -s + pytest tests/e2e/features/custom_pipeline/test_async_omni_collective_rpc.py -v -s """ from __future__ import annotations @@ -34,12 +34,8 @@ from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.outputs import OmniRequestOutput MODEL = "tiny-random/Qwen-Image" -CUSTOM_PIPELINE_CLASS = ( - "tests.e2e.offline_inference.custom_pipeline.qwen_image_pipeline_with_logprob.QwenImagePipelineWithLogProbForTest" -) -WORKER_EXTENSION_CLASS = ( - "tests.e2e.offline_inference.custom_pipeline.worker_extension.vLLMOmniColocateWorkerExtensionForTest" -) +CUSTOM_PIPELINE_CLASS = "tests.e2e.features.helpers.custom_pipeline.QwenImagePipelineWithLogProbForTest" +WORKER_EXTENSION_CLASS = "tests.e2e.features.helpers.custom_pipeline.vLLMOmniColocateWorkerExtensionForTest" def _create_inline_engine() -> AsyncOmni: diff --git a/tests/e2e/offline_inference/custom_pipeline/test_async_omni_qwen_image_generate.py b/tests/e2e/features/custom_pipeline/test_async_omni_qwen_image_generate.py similarity index 96% rename from tests/e2e/offline_inference/custom_pipeline/test_async_omni_qwen_image_generate.py rename to tests/e2e/features/custom_pipeline/test_async_omni_qwen_image_generate.py index 02519f90f..b24f364e4 100644 --- a/tests/e2e/offline_inference/custom_pipeline/test_async_omni_qwen_image_generate.py +++ b/tests/e2e/features/custom_pipeline/test_async_omni_qwen_image_generate.py @@ -24,12 +24,8 @@ from vllm_omni.entrypoints.async_omni import AsyncOmni from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.outputs import OmniRequestOutput -CUSTOM_PIPELINE_CLASS = ( - "tests.e2e.offline_inference.custom_pipeline.qwen_image_pipeline_with_logprob.QwenImagePipelineWithLogProbForTest" -) -WORKER_EXTENSION_CLASS = ( - "tests.e2e.offline_inference.custom_pipeline.worker_extension.vLLMOmniColocateWorkerExtensionForTest" -) +CUSTOM_PIPELINE_CLASS = "tests.e2e.features.helpers.custom_pipeline.QwenImagePipelineWithLogProbForTest" +WORKER_EXTENSION_CLASS = "tests.e2e.features.helpers.custom_pipeline.vLLMOmniColocateWorkerExtensionForTest" # Use your specified HF repo name/path directly here MODEL = "tiny-random/Qwen-Image" diff --git a/tests/e2e/offline_inference/custom_pipeline/test_worker_extension.py b/tests/e2e/features/custom_pipeline/test_worker_extension.py similarity index 59% rename from tests/e2e/offline_inference/custom_pipeline/test_worker_extension.py rename to tests/e2e/features/custom_pipeline/test_worker_extension.py index 16dddde40..da7b0ac17 100644 --- a/tests/e2e/offline_inference/custom_pipeline/test_worker_extension.py +++ b/tests/e2e/features/custom_pipeline/test_worker_extension.py @@ -7,37 +7,19 @@ from contextlib import ExitStack import pytest -from tests.e2e.offline_inference.custom_pipeline.qwen_image_pipeline_with_logprob import ( +from tests.e2e.features.helpers.custom_pipeline import ( QwenImagePipelineWithLogProbForTest, ) -from tests.e2e.offline_inference.custom_pipeline.worker_extension import ( - vLLMOmniColocateWorkerExtensionForTest, -) from tests.helpers.mark import hardware_test -from vllm_omni.diffusion.worker.diffusion_worker import CustomPipelineWorkerExtension from vllm_omni.entrypoints.async_omni import AsyncOmni CUSTOM_PIPELINE_CLASS = QwenImagePipelineWithLogProbForTest -WORKER_EXTENSION_CLASS = ( - "tests.e2e.offline_inference.custom_pipeline.worker_extension.vLLMOmniColocateWorkerExtensionForTest" -) +WORKER_EXTENSION_CLASS = "tests.e2e.features.helpers.custom_pipeline.vLLMOmniColocateWorkerExtensionForTest" MODEL = "tiny-random/Qwen-Image" pytestmark = [pytest.mark.core_model] -@pytest.mark.cpu -def test_worker_extension_inheritance(): - assert issubclass(vLLMOmniColocateWorkerExtensionForTest, CustomPipelineWorkerExtension) - - -@pytest.mark.cpu -def test_worker_extension_test_function(): - assert ( - vLLMOmniColocateWorkerExtensionForTest.test_extension_name() == "vllm-omni-colocate-worker-extension-for-test" - ) - - @pytest.mark.diffusion @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.asyncio diff --git a/tests/fullduplex/engine/test_duplex_control_client.py b/tests/e2e/features/fullduplex/engine/test_duplex_control_client.py similarity index 100% rename from tests/fullduplex/engine/test_duplex_control_client.py rename to tests/e2e/features/fullduplex/engine/test_duplex_control_client.py diff --git a/tests/fullduplex/engine/test_duplex_control_plane.py b/tests/e2e/features/fullduplex/engine/test_duplex_control_plane.py similarity index 100% rename from tests/fullduplex/engine/test_duplex_control_plane.py rename to tests/e2e/features/fullduplex/engine/test_duplex_control_plane.py diff --git a/tests/fullduplex/engine/test_duplex_deploy_config.py b/tests/e2e/features/fullduplex/engine/test_duplex_deploy_config.py similarity index 100% rename from tests/fullduplex/engine/test_duplex_deploy_config.py rename to tests/e2e/features/fullduplex/engine/test_duplex_deploy_config.py diff --git a/tests/fullduplex/engine/test_duplex_intermediate.py b/tests/e2e/features/fullduplex/engine/test_duplex_intermediate.py similarity index 100% rename from tests/fullduplex/engine/test_duplex_intermediate.py rename to tests/e2e/features/fullduplex/engine/test_duplex_intermediate.py diff --git a/tests/fullduplex/engine/test_duplex_lease.py b/tests/e2e/features/fullduplex/engine/test_duplex_lease.py similarity index 100% rename from tests/fullduplex/engine/test_duplex_lease.py rename to tests/e2e/features/fullduplex/engine/test_duplex_lease.py diff --git a/tests/fullduplex/engine/test_duplex_runtime.py b/tests/e2e/features/fullduplex/engine/test_duplex_runtime.py similarity index 100% rename from tests/fullduplex/engine/test_duplex_runtime.py rename to tests/e2e/features/fullduplex/engine/test_duplex_runtime.py diff --git a/tests/fullduplex/minicpmo45/__init__.py b/tests/e2e/features/fullduplex/minicpmo45/__init__.py similarity index 100% rename from tests/fullduplex/minicpmo45/__init__.py rename to tests/e2e/features/fullduplex/minicpmo45/__init__.py diff --git a/tests/fullduplex/minicpmo45/test_commit_policy.py b/tests/e2e/features/fullduplex/minicpmo45/test_commit_policy.py similarity index 100% rename from tests/fullduplex/minicpmo45/test_commit_policy.py rename to tests/e2e/features/fullduplex/minicpmo45/test_commit_policy.py diff --git a/tests/fullduplex/minicpmo45/test_input.py b/tests/e2e/features/fullduplex/minicpmo45/test_input.py similarity index 100% rename from tests/fullduplex/minicpmo45/test_input.py rename to tests/e2e/features/fullduplex/minicpmo45/test_input.py diff --git a/tests/fullduplex/openai/test_duplex_audio.py b/tests/e2e/features/fullduplex/openai/test_duplex_audio.py similarity index 100% rename from tests/fullduplex/openai/test_duplex_audio.py rename to tests/e2e/features/fullduplex/openai/test_duplex_audio.py diff --git a/tests/fullduplex/openai/test_runtime_adapter_boundary.py b/tests/e2e/features/fullduplex/openai/test_runtime_adapter_boundary.py similarity index 100% rename from tests/fullduplex/openai/test_runtime_adapter_boundary.py rename to tests/e2e/features/fullduplex/openai/test_runtime_adapter_boundary.py diff --git a/tests/fullduplex/openai/test_websocket_actor.py b/tests/e2e/features/fullduplex/openai/test_websocket_actor.py similarity index 100% rename from tests/fullduplex/openai/test_websocket_actor.py rename to tests/e2e/features/fullduplex/openai/test_websocket_actor.py diff --git a/tests/fullduplex/test_client.py b/tests/e2e/features/fullduplex/test_client.py similarity index 100% rename from tests/fullduplex/test_client.py rename to tests/e2e/features/fullduplex/test_client.py diff --git a/tests/fullduplex/test_joyvl_brain.py b/tests/e2e/features/fullduplex/test_joyvl_brain.py similarity index 100% rename from tests/fullduplex/test_joyvl_brain.py rename to tests/e2e/features/fullduplex/test_joyvl_brain.py diff --git a/tests/fullduplex/test_joyvl_policy.py b/tests/e2e/features/fullduplex/test_joyvl_policy.py similarity index 100% rename from tests/fullduplex/test_joyvl_policy.py rename to tests/e2e/features/fullduplex/test_joyvl_policy.py diff --git a/tests/fullduplex/test_runtime.py b/tests/e2e/features/fullduplex/test_runtime.py similarity index 100% rename from tests/fullduplex/test_runtime.py rename to tests/e2e/features/fullduplex/test_runtime.py diff --git a/tests/e2e/offline_inference/custom_pipeline/qwen_image_pipeline_with_logprob.py b/tests/e2e/features/helpers/custom_pipeline.py similarity index 59% rename from tests/e2e/offline_inference/custom_pipeline/qwen_image_pipeline_with_logprob.py rename to tests/e2e/features/helpers/custom_pipeline.py index 193d1b1e2..63cb21056 100644 --- a/tests/e2e/offline_inference/custom_pipeline/qwen_image_pipeline_with_logprob.py +++ b/tests/e2e/features/helpers/custom_pipeline.py @@ -1,27 +1,299 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Detailed custom Qwen-Image pipeline for E2E tests. +"""Test doubles for custom-pipeline E2E and RLHF integration. -This pipeline follows the structure of the user's reference implementation: -- supports pre-tokenized prompt IDs via OmniCustomPrompt-style dict input -- uses an SDE scheduler that can return step logprobs -- returns structured trajectory_* fields (latents, timesteps, log_probs) - consistent with the BAGEL trajectory recording design +Three layers: + +* ``FlowMatchSDEDiscreteSchedulerForTest`` — SDE scheduler with per-step logprobs. +* ``vLLMOmniColocateWorkerExtensionForTest`` — verl-omni colocate worker extension mirror. +* ``QwenImagePipelineWithLogProbForTest`` — Qwen-Image pipeline returning trajectory_* fields. """ from __future__ import annotations +import ctypes +import logging +import math import os +import platform +import signal +from dataclasses import dataclass from typing import Any, Literal import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from diffusers.utils import BaseOutput +from diffusers.utils.torch_utils import randn_tensor +from msgspec import field -from tests.e2e.offline_inference.custom_pipeline.flow_match_sde_scheduler import FlowMatchSDEDiscreteSchedulerForTest from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig from vllm_omni.diffusion.distributed.utils import get_local_device from vllm_omni.diffusion.models.qwen_image import QwenImagePipeline +from vllm_omni.diffusion.worker.diffusion_worker import CustomPipelineWorkerExtension from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch +from vllm_omni.lora.request import LoRARequest as OmniLoRARequest + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Scheduler layer +# --------------------------------------------------------------------------- + + +@dataclass +class FlowMatchSDEDiscreteSchedulerOutputForTest(BaseOutput): + """ + Output class for the scheduler's `step` function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_channels)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + log_prob (`torch.FloatTensor` of shape `(batch_size,)`, *optional*): + The log probability of the previous sample. + prev_sample_mean (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_channels)` for images): + The mean of the computed sample of previous timestep. + std_dev_t (`torch.FloatTensor` of shape `(batch_size, 1, 1)`): + The standard deviation used to compute `prev_sample`. + """ + + prev_sample: torch.FloatTensor + log_prob: torch.FloatTensor | None + prev_sample_mean: torch.FloatTensor + std_dev_t: torch.FloatTensor + + +class FlowMatchSDEDiscreteSchedulerForTest(FlowMatchEulerDiscreteScheduler): + """SDE version of the FlowMatchEulerDiscreteScheduler. + The implementation is based on FlowGRPO paper (https://arxiv.org/abs/2505.05470) + and diffusers v0.37 branch. + """ + + def step( + self, + model_output: torch.FloatTensor, + timestep: float | torch.FloatTensor, + sample: torch.FloatTensor, + s_churn: float = 0.0, + s_tmin: float = 0.0, + s_tmax: float = float("inf"), + s_noise: float = 1.0, + generator: torch.Generator | None = None, + per_token_timesteps: torch.Tensor | None = None, + return_dict: bool = True, + noise_level: float = 0.7, + prev_sample: torch.FloatTensor | None = None, + sde_type: Literal["sde", "cps"] = "sde", + return_logprobs: bool = True, + ) -> FlowMatchSDEDiscreteSchedulerOutputForTest | tuple: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Modified from https://github.com/yifan123/flow_grpo/blob/main/flow_grpo/diffusers_patch/sd3_sde_with_logprob.py + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`float`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + s_churn (`float`): + s_tmin (`float`): + s_tmax (`float`): + s_noise (`float`, defaults to 1.0): + Scaling factor for noise added to the sample. + generator (`torch.Generator`, *optional*): + A random number generator. + per_token_timesteps (`torch.Tensor`, *optional*): + The timesteps for each token in the sample. + return_dict (`bool`): + Whether or not to return a + [`~schedulers.scheduling_flow_match_euler_discrete.FlowMatchSDEDiscreteSchedulerOutput`] or tuple. + noise_level (`float`, *optional*, defaults to 0.7): + The noise level used in the SDE. + prev_sample (`torch.FloatTensor`, *optional*): + The sample from the previous timestep. If not provided, it will be sampled inside the function. + sde_type (`str`, *optional*, defaults to "sde"): + The type of SDE to use. Choose between "sde" and "cps". + return_logprobs (`bool`, *optional*, defaults to True): + Whether to return log probabilities of the previous sample. + """ + + if isinstance(timestep, (int, torch.IntTensor, torch.LongTensor)): + raise ValueError( + ( + "Passing integer indices (e.g. from `enumerate(timesteps)`) as timesteps to" + " `FlowMatchEulerDiscreteScheduler.step()` is not supported. Make sure to pass" + " one of the `scheduler.timesteps` as a timestep." + ), + ) + + if self.step_index is None: + self._init_step_index(timestep) + + # Upcast to avoid precision issues when computing prev_sample + sample = sample.to(torch.float32) + if prev_sample is not None: + prev_sample = prev_sample.to(torch.float32) + + prev_sample, log_prob, prev_sample_mean, std_dev_t = self.sample_previous_step( + sample=sample, + model_output=model_output, + generator=generator, + per_token_timesteps=per_token_timesteps, + noise_level=noise_level, + prev_sample=prev_sample, + sde_type=sde_type, + return_logprobs=return_logprobs, + ) + + # upon completion increase step index by one + self._step_index += 1 + if per_token_timesteps is None: + # Cast sample back to model compatible dtype + prev_sample = prev_sample.to(model_output.dtype) + + if not return_dict: + return (prev_sample, log_prob, prev_sample_mean, std_dev_t) + + return FlowMatchSDEDiscreteSchedulerOutputForTest( + prev_sample=prev_sample, log_prob=log_prob, prev_sample_mean=prev_sample_mean, std_dev_t=std_dev_t + ) + + def sample_previous_step( + self, + sample: torch.Tensor, + model_output: torch.Tensor, + timestep: torch.FloatTensor | None = None, + generator: torch.Generator | None = None, + per_token_timesteps: torch.Tensor | None = None, + noise_level: float = 0.7, + prev_sample: torch.Tensor | None = None, + sde_type: Literal["cps", "sde"] = "sde", + return_logprobs: bool = True, + ): + assert sde_type in ["sde", "cps"] + assert sample.dtype == torch.float32 + if prev_sample is not None: + assert prev_sample.dtype == torch.float32 + + if per_token_timesteps is not None: + raise NotImplementedError("per_token_timesteps is not supported yet for FlowMatchSDEDiscreteScheduler.") + else: + if timestep is None: + sigma_idx = self.step_index + sigma = self.sigmas[sigma_idx] + sigma_prev = self.sigmas[sigma_idx + 1] + else: + sigma_idx = torch.tensor([self.index_for_timestep(t) for t in timestep]) + sigma = self.sigmas[sigma_idx].view(-1, *([1] * (len(sample.shape) - 1))) + sigma_prev = self.sigmas[sigma_idx + 1].view(-1, *([1] * (len(sample.shape) - 1))) + + sigma_max = self.sigmas[1] + dt = sigma_prev - sigma + + if sde_type == "sde": + std_dev_t = torch.sqrt(sigma / (1 - torch.where(sigma == 1, sigma_max, sigma))) * noise_level + + prev_sample_mean = ( + sample * (1 + std_dev_t**2 / (2 * sigma) * dt) + + model_output * (1 + std_dev_t**2 * (1 - sigma) / (2 * sigma)) * dt + ) + + if prev_sample is None: + variance_noise = randn_tensor( + model_output.shape, + generator=generator, + device=model_output.device, + dtype=model_output.dtype, + ) + prev_sample = prev_sample_mean + std_dev_t * torch.sqrt(-1 * dt) * variance_noise + + if return_logprobs: + log_prob = ( + -((prev_sample.detach() - prev_sample_mean) ** 2) / (2 * ((std_dev_t * torch.sqrt(-1 * dt)) ** 2)) + - torch.log(std_dev_t * torch.sqrt(-1 * dt)) + - torch.log(torch.sqrt(2 * torch.as_tensor(math.pi))) + ) + else: + log_prob = None + + elif sde_type == "cps": + std_dev_t = sigma_prev * math.sin(noise_level * math.pi / 2) + pred_original_sample = sample - sigma * model_output + noise_estimate = sample + model_output * (1 - sigma) + prev_sample_mean = pred_original_sample * (1 - sigma_prev) + noise_estimate * torch.sqrt( + sigma_prev**2 - std_dev_t**2 + ) + + if prev_sample is None: + variance_noise = randn_tensor( + model_output.shape, + generator=generator, + device=model_output.device, + dtype=model_output.dtype, + ) + prev_sample = prev_sample_mean + std_dev_t * variance_noise + + if return_logprobs: + log_prob = -((prev_sample.detach() - prev_sample_mean) ** 2) + else: + log_prob = None + + # mean along all but batch dimension + log_prob = log_prob.mean(dim=tuple(range(1, log_prob.ndim))) if log_prob is not None else None + return prev_sample, log_prob, prev_sample_mean, std_dev_t + + +# --------------------------------------------------------------------------- +# Worker extension layer +# --------------------------------------------------------------------------- + + +class OmniTensorLoRARequestForTest(OmniLoRARequest): + peft_config: dict = field(default=None) + lora_tensors: dict = field(default=None) + + +def set_death_signal() -> None: + """Verbatim port of ``verl.workers.rollout.vllm_rollout.utils.set_death_signal``.""" + if platform.system() != "Linux": + return + try: + libc = ctypes.CDLL("libc.so.6") + libc.prctl(1, signal.SIGKILL) + if os.getppid() == 1: + os.kill(os.getpid(), signal.SIGKILL) + except Exception: # noqa: BLE001 + # libc.so.6 may not be present (musl, alpine) — best-effort only. + pass + + +class vLLMOmniColocateWorkerExtensionForTest(CustomPipelineWorkerExtension): + """Mirror of ``vLLMOmniColocateWorkerExtension`` (verl-omni). + + The production ``__new__`` runs ``set_death_signal`` + ``VLLMOmniHijack.hijack`` + on every vLLM worker process. Replicating both ensures this test + reproduces the same monkey-patched environment. + """ + + def __new__(cls, **kwargs): + set_death_signal() + return super().__new__(cls) + + @staticmethod + def test_extension_name() -> str: + """Return a stable identifier for assertions in unit tests.""" + return "vllm-omni-colocate-worker-extension-for-test" + + +# --------------------------------------------------------------------------- +# Pipeline layer +# --------------------------------------------------------------------------- def _maybe_to_cpu(v): @@ -30,8 +302,6 @@ def _maybe_to_cpu(v): return v -# Custom pipeline class for QwenImage that returns log probabilities during the diffusion process. -# This is for test class QwenImagePipelineWithLogProbForTest(QwenImagePipeline): supports_request_batch = False diff --git a/tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py b/tests/e2e/features/helpers/verl_omni_server.py similarity index 52% rename from tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py rename to tests/e2e/features/helpers/verl_omni_server.py index 420c9bb68..d53e0936f 100644 --- a/tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py +++ b/tests/e2e/features/helpers/verl_omni_server.py @@ -1,30 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""E2E test that follows the EXACT same flow as -``verl-omni/tests/workers/rollout/rollout_vllm/test_vllm_omni_generate.py`` -but inlines all the ``verl`` / ``verl_omni`` symbols the flow depends on -(stripped down to only what the diffusion-mode test actually exercises). +"""Ray-serializable verl-omni HTTP server test double for RLHF E2E tests. -Flow (identical to the verl-omni test): - -1. Build two ``OmegaConf`` configs (``DiffusionRolloutConfig`` / - ``DiffusionModelConfig`` shape). -2. ``ServerCls = ray.remote(vLLMOmniHttpServerLocal)`` then - ``ServerCls.options(...).remote(config=..., model_config=..., - rollout_mode=RolloutMode.STANDALONE, workers=[], replica_rank=0, - node_rank=0, gpus_per_node=1, nnodes=1, cuda_visible_devices="0")``. -3. ``ray.get(server.launch_server.remote())``. -4. ``server.generate.remote(prompt_ids=..., sampling_params=..., request_id=...)`` - and assert the returned ``DiffusionOutput``. - -The inlined classes / helpers mirror their ``verl`` / ``verl_omni`` -counterparts; bits not reached by this test (AR mode, LoRA, -quantization, run_uvicorn, headless, profiler, port wiring) are removed. - -Usage: - pytest tests/e2e/offline_inference/rlhf_test/\ -test_ray_async_omni_qwen_image_generate.py -v -s +The actor class must live in an importable module (not a pytest test file) so +Ray workers can unpickle ``vLLMOmniHttpServerLocal`` via +``tests.e2e.features.helpers.verl_omni_server``. """ from __future__ import annotations @@ -35,23 +16,15 @@ import logging import os from dataclasses import asdict from enum import Enum -from functools import lru_cache from typing import Any -from uuid import uuid4 -import pytest -import ray -import torch import torchvision.transforms as T -from huggingface_hub import snapshot_download from omegaconf import DictConfig, OmegaConf from pydantic import BaseModel, ConfigDict -from transformers import AutoTokenizer from vllm.entrypoints.openai.api_server import build_app from vllm.utils.argparse_utils import FlexibleArgumentParser import vllm_omni.entrypoints.cli.serve -from tests.helpers.mark import hardware_test from vllm_omni.engine.arg_utils import OmniEngineArgs from vllm_omni.entrypoints.async_omni import AsyncOmni from vllm_omni.entrypoints.openai.api_server import omni_init_app_state @@ -60,33 +33,8 @@ from vllm_omni.outputs import OmniRequestOutput logger = logging.getLogger(__name__) -CUSTOM_PIPELINE_CLASS = ( - "tests.e2e.offline_inference.custom_pipeline.qwen_image_pipeline_with_logprob.QwenImagePipelineWithLogProbForTest" -) -WORKER_EXTENSION_CLASS = ( - "tests.e2e.offline_inference.custom_pipeline.worker_extension.vLLMOmniColocateWorkerExtensionForTest" -) - -# Use your specified HF repo name/path directly here -MODEL = "tiny-random/Qwen-Image" - -TOKENIZER_MODEL = "Qwen/Qwen2-1.5B-Instruct" - - -def _resolve_model_path(repo_id: str) -> str: - """Resolve an HF repo ID to a local snapshot path, downloading if needed.""" - # Allow overriding with a pre-existing local path (skips download). - if os.path.isdir(repo_id): - return repo_id - return snapshot_download(repo_id=repo_id) - - -_MIN_PROMPT_TOKENS = 35 - - -# --------------------------------------------------------------------- -# Inlined: verl.workers.rollout.replica.RolloutMode -# --------------------------------------------------------------------- +CUSTOM_PIPELINE_CLASS = "tests.e2e.features.helpers.custom_pipeline.QwenImagePipelineWithLogProbForTest" +WORKER_EXTENSION_CLASS = "tests.e2e.features.helpers.custom_pipeline.vLLMOmniColocateWorkerExtensionForTest" class RolloutMode(Enum): @@ -95,11 +43,6 @@ class RolloutMode(Enum): STANDALONE = "standalone" -# --------------------------------------------------------------------- -# Inlined: verl_omni.workers.rollout.replica.DiffusionOutput -# --------------------------------------------------------------------- - - class DiffusionOutput(BaseModel): """Pydantic mirror of ``verl_omni.workers.rollout.replica.DiffusionOutput``.""" @@ -112,11 +55,6 @@ class DiffusionOutput(BaseModel): extra_fields: dict[str, Any] = {} -# --------------------------------------------------------------------- -# Inlined: verl.utils.tokenizer.normalize_token_ids -# --------------------------------------------------------------------- - - def normalize_token_ids(tokenized_output) -> list[int]: token_ids = tokenized_output if isinstance(tokenized_output, dict): @@ -142,11 +80,6 @@ def normalize_token_ids(tokenized_output) -> list[int]: return out -# --------------------------------------------------------------------- -# Inlined: verl.workers.rollout.vllm_rollout.utils.build_cli_args_from_config -# --------------------------------------------------------------------- - - def build_cli_args_from_config(config: dict[str, Any]) -> list[str]: cli_args: list[str] = [] for k, v in config.items(): @@ -166,34 +99,6 @@ def build_cli_args_from_config(config: dict[str, Any]) -> list[str]: return cli_args -# --------------------------------------------------------------------- -# Tokenizer helper (test-side) -# --------------------------------------------------------------------- - - -@lru_cache(maxsize=1) -def _get_tokenizer(): - return AutoTokenizer.from_pretrained(TOKENIZER_MODEL, trust_remote_code=True) - - -def _tokenize_prompt(text: str) -> list[int]: - tokenizer = _get_tokenizer() - messages = [{"role": "user", "content": text}] - token_ids = normalize_token_ids(tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=False)) - assert len(token_ids) > _MIN_PROMPT_TOKENS, ( - f"Prompt too short ({len(token_ids)} tokens, need >{_MIN_PROMPT_TOKENS}). " - "The pipeline drops the first 34 chat-template prefix tokens; " - "use a longer prompt so content tokens remain after the drop." - ) - return token_ids - - -# --------------------------------------------------------------------- -# Inlined: vLLMOmniHttpServer (verl-omni) + relevant vLLMHttpServer (verl) -# bits, stripped to the diffusion-mode path the test exercises. -# --------------------------------------------------------------------- - - def _cfg_get(cfg: Any, key: str, default: Any = None) -> Any: """Uniform getter that works for both DictConfig and dataclass-like objects.""" if isinstance(cfg, DictConfig): @@ -202,13 +107,8 @@ def _cfg_get(cfg: Any, key: str, default: Any = None) -> Any: class vLLMOmniHttpServerLocal: - """Self-contained simulation of ``vLLMOmniHttpServer`` for diffusion mode. + """Self-contained simulation of ``vLLMOmniHttpServer`` for diffusion mode.""" - Constructor signature matches ``vLLMHttpServer.__init__`` exactly so the - Ray-actor call site in the test is identical to the verl-omni one. - """ - - # ------------------------------------------------------------------ init def __init__( self, config, @@ -224,8 +124,6 @@ class vLLMOmniHttpServerLocal: os.environ.setdefault("CUDA_VISIBLE_DEVICES", cuda_visible_devices) os.environ["VERL_REPLICA_RANK"] = str(replica_rank) - # Resolve configs. verl uses omega_conf_to_dataclass; here we keep - # them as DictConfig — every read site below uses ``_cfg_get``. self.config = OmegaConf.create(config) if not isinstance(config, DictConfig) else config self.model_config = OmegaConf.create(model_config) if not isinstance(model_config, DictConfig) else model_config @@ -237,8 +135,6 @@ class vLLMOmniHttpServerLocal: self.nnodes = nnodes self.global_steps = 0 self.engine: AsyncOmni | None = None - - # Diffusion-mode parity with ``vLLMOmniHttpServer._post_init``. self._to_tensor = T.PILToTensor() logger.info( @@ -249,11 +145,7 @@ class vLLMOmniHttpServerLocal: cuda_visible_devices, ) - # ---------------------------------------------------------- launch_server async def launch_server(self) -> None: - """Diffusion-only mirror of ``vLLMHttpServer.launch_server``.""" - - # vLLMOmniHttpServer._get_engine_kwargs_key() returns "vllm_omni". engine_kwargs_root = _cfg_get(self.config, "engine_kwargs", {}) or {} if isinstance(engine_kwargs_root, (dict, DictConfig)): engine_kwargs = engine_kwargs_root.get("vllm_omni", {}) or {} @@ -261,10 +153,8 @@ class vLLMOmniHttpServerLocal: engine_kwargs = {} engine_kwargs = {k: v for k, v in dict(engine_kwargs).items() if v is not None} - # vLLMOmniHttpServer._preprocess_engine_kwargs (diffusion path). engine_kwargs.pop("output_mode", None) - # ``compilation_config`` handling — verbatim from parent launch_server. compilation_config = engine_kwargs.pop("compilation_config", None) or {} if isinstance(compilation_config, str): compilation_config = json.loads(compilation_config) @@ -273,8 +163,6 @@ class vLLMOmniHttpServerLocal: compilation_config.setdefault("cudagraph_mode", "FULL_AND_PIECEWISE") compilation_config = json.dumps(compilation_config) - # Args dict (subset of parent's; the diffusion override drops AR-only - # entries via ``_get_override_generation_config`` returning ``{}``). args = { "dtype": _cfg_get(self.config, "dtype", "bfloat16"), "load_format": _cfg_get(self.config, "load_format", "auto"), @@ -303,8 +191,6 @@ class vLLMOmniHttpServerLocal: model_path = str(_cfg_get(self.model_config, "local_path", None) or _cfg_get(self.model_config, "path")) server_args_list = ["serve", model_path] + build_cli_args_from_config(args) - # Same parser pipeline as parent launch_server (with vllm-omni's - # cmd_init injected, matching ``_get_cli_modules`` + ``_get_cli_description``). parser = FlexibleArgumentParser(description="vLLM-Omni CLI") subparsers = parser.add_subparsers(required=False, dest="subparser") cmds: dict[str, Any] = {} @@ -316,19 +202,12 @@ class vLLMOmniHttpServerLocal: if parsed.subparser in cmds: cmds[parsed.subparser].validate(parsed) - # node_rank == 0 → run_server (the test never sets node_rank != 0). await self.run_server(parsed) - # ------------------------------------------------------------- run_server async def run_server(self, args: argparse.Namespace) -> None: - """Diffusion branch of ``vLLMOmniHttpServer.run_server``.""" - engine_args = OmniEngineArgs.from_cli_args(args) engine_args = asdict(engine_args) - # Diffusion path: pin the in-repo test pipeline. The production code - # calls ``VllmOmniPipelineBase.get_pipeline_path(architecture)``; we - # bypass the registry and bind the test pipeline directly. engine_args["enable_dummy_pipeline"] = True engine_args["custom_pipeline_args"] = {"pipeline_class": CUSTOM_PIPELINE_CLASS} @@ -337,10 +216,7 @@ class vLLMOmniHttpServerLocal: await omni_init_app_state(engine_client, app.state, args) self.engine = engine_client - # ``run_uvicorn`` intentionally skipped — the test drives the engine - # via Ray RPC, not HTTP. - # ---------------------------------------------------------------- generate async def generate( self, prompt_ids: list[int], @@ -351,8 +227,6 @@ class vLLMOmniHttpServerLocal: negative_prompt_ids: list[int] | None = None, priority: int = 0, # noqa: ARG002 (signature parity) ) -> DiffusionOutput: - # Verbatim copy of ``vLLMOmniHttpServer._generate_diffusion`` minus - # the LoRA branch (the test does not enable ``lora_as_adapter``). prompt_ids = normalize_token_ids(prompt_ids) multi_modal_data: dict[str, Any] = {} @@ -444,149 +318,3 @@ class vLLMOmniHttpServerLocal: num_preempted=num_preempted, extra_fields=extra_fields, ) - - -# --------------------------------------------------------------------- -# Fixtures & tests -# --------------------------------------------------------------------- - - -@pytest.fixture -def init_server(): - """Create and launch a vLLMOmniHttpServerLocal Ray actor with Qwen-Image.""" - model_path = _resolve_model_path(MODEL) - - ray.init( - runtime_env={ - "env_vars": { - "TOKENIZERS_PARALLELISM": "true", - "NCCL_DEBUG": "WARN", - "VLLM_LOGGING_LEVEL": "INFO", - } - }, - ignore_reinit_error=True, - ) - - rollout_cfg = OmegaConf.create( - { - "name": "vllm_omni", - "mode": "async", - "tensor_model_parallel_size": 1, - "data_parallel_size": 1, - "pipeline_model_parallel_size": 1, - "gpu_memory_utilization": 0.8, - "max_num_batched_tokens": 8192, - # The custom log-prob pipeline only supports serial requests. - "max_num_seqs": 1, - "max_model_len": 1058, - "dtype": "bfloat16", - "load_format": "auto", - "enforce_eager": True, - "enable_chunked_prefill": False, - "enable_prefix_caching": False, - "enable_sleep_mode": False, - "free_cache_engine": True, - "disable_log_stats": True, - "logprobs_mode": "processed_logprobs", - "scheduling_policy": "fcfs", - "seed": 0, - "n": 4, - "pipeline": { - "height": 512, - "width": 512, - "num_inference_steps": 10, - }, - } - ) - - model_cfg = OmegaConf.create( - { - "path": str(model_path), - "local_path": str(model_path), - "tokenizer_path": TOKENIZER_MODEL, - "trust_remote_code": True, - "load_tokenizer": True, - } - ) - - ServerCls = ray.remote(vLLMOmniHttpServerLocal) - server = ServerCls.options( - runtime_env={ - "env_vars": { - "RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES": "1", - "RAY_EXPERIMENTAL_NOSET_ASCEND_RT_VISIBLE_DEVICES": "1", - "NCCL_CUMEM_ENABLE": "0", - } - }, - max_concurrency=10, - num_gpus=1, - ).remote( - config=rollout_cfg, - model_config=model_cfg, - rollout_mode=RolloutMode.STANDALONE, - workers=[], - replica_rank=0, - node_rank=0, - gpus_per_node=1, - nnodes=1, - cuda_visible_devices="0", - ) - - ray.get(server.launch_server.remote()) - - yield server - - ray.shutdown() - - -@pytest.mark.core_model -@pytest.mark.diffusion -@hardware_test(res={"cuda": "L4"}, num_cards=1) -def test_generate(init_server): - """Concurrent generate() calls covering basic output, logprobs, and multi-request correctness.""" - server = init_server - - prompts = [ - "a beautiful sunset over the ocean with vibrant orange and purple clouds " - "reflecting on the calm water surface near a rocky coastline", - "a fluffy orange cat sitting on a wooden windowsill looking outside at " - "a garden full of colorful flowers on a bright sunny afternoon", - "a majestic mountain landscape covered with fresh white snow under a " - "clear blue sky with pine trees in the foreground and a frozen lake", - "a futuristic city at night with neon lights glowing on tall glass " - "skyscrapers and flying vehicles soaring between the buildings", - ] - - refs = [] - for i, prompt in enumerate(prompts): - rid = f"test_{i}_{uuid4().hex[:8]}" - ref = server.generate.remote( - prompt_ids=_tokenize_prompt(prompt), - sampling_params={ - "num_inference_steps": 10, - "true_cfg_scale": 4.0, - "height": 512, - "width": 512, - "logprobs": i == 0, # first request includes logprobs - }, - request_id=rid, - ) - refs.append(ref) - - results = ray.get(refs, timeout=600) - - for i, output in enumerate(results): - assert isinstance(output, DiffusionOutput), f"Request {i}: expected DiffusionOutput" - img = output.diffusion_output - assert isinstance(img, torch.Tensor) and img.ndim == 3, ( - f"Request {i}: expected CHW torch.Tensor, got {type(img).__name__} ndim={getattr(img, 'ndim', None)}" - ) - c, h, w = img.shape - assert c == 3, f"Request {i}: expected 3 channels (CHW), got {c}" - assert h > 0 and w > 0, f"Request {i}: image dimensions must be positive" - assert output.stop_reason in ("completed", "aborted", None), ( - f"Request {i}: unexpected stop_reason {output.stop_reason!r}" - ) - assert 0.0 <= float(img[0, 0, 0]) <= 1.0, f"Request {i}: pixel values must be in [0, 1]" - - print(f"All {len(prompts)} concurrent requests returned valid DiffusionOutput") diff --git a/tests/e2e/features/rlhf_test/test_verl_omni_e2e.py b/tests/e2e/features/rlhf_test/test_verl_omni_e2e.py new file mode 100644 index 000000000..dab5feef1 --- /dev/null +++ b/tests/e2e/features/rlhf_test/test_verl_omni_e2e.py @@ -0,0 +1,217 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""E2E test that follows the EXACT same flow as +``verl-omni/tests/workers/rollout/rollout_vllm/test_vllm_omni_generate.py`` +but inlines all the ``verl`` / ``verl_omni`` symbols the flow depends on +(stripped down to only what the diffusion-mode test actually exercises). + +Flow (identical to the verl-omni test): + +1. Build two ``OmegaConf`` configs (``DiffusionRolloutConfig`` / + ``DiffusionModelConfig`` shape). +2. ``ServerCls = ray.remote(vLLMOmniHttpServerLocal)`` then + ``ServerCls.options(...).remote(config=..., model_config=..., + rollout_mode=RolloutMode.STANDALONE, workers=[], replica_rank=0, + node_rank=0, gpus_per_node=1, nnodes=1, cuda_visible_devices="0")``. +3. ``ray.get(server.launch_server.remote())``. +4. ``server.generate.remote(prompt_ids=..., sampling_params=..., request_id=...)`` + and assert the returned ``DiffusionOutput``. + +The Ray actor lives in ``tests.e2e.features.helpers.verl_omni_server`` so +workers can import it after directory migration (pytest test modules are not +importable as ``test_verl_omni_e2e`` in Ray child processes). + +Usage: + pytest tests/e2e/features/rlhf_test/test_verl_omni_e2e.py -v -s +""" + +from __future__ import annotations + +import os +from functools import lru_cache +from uuid import uuid4 + +import pytest +import ray +import torch +from huggingface_hub import snapshot_download +from omegaconf import OmegaConf +from transformers import AutoTokenizer + +from tests.e2e.features.helpers.verl_omni_server import ( + DiffusionOutput, + RolloutMode, + normalize_token_ids, + vLLMOmniHttpServerLocal, +) +from tests.helpers.mark import hardware_test + +MODEL = "tiny-random/Qwen-Image" +TOKENIZER_MODEL = "Qwen/Qwen2-1.5B-Instruct" +_MIN_PROMPT_TOKENS = 35 + + +def _resolve_model_path(repo_id: str) -> str: + if os.path.isdir(repo_id): + return repo_id + return snapshot_download(repo_id=repo_id) + + +@lru_cache(maxsize=1) +def _get_tokenizer(): + return AutoTokenizer.from_pretrained(TOKENIZER_MODEL, trust_remote_code=True) + + +def _tokenize_prompt(text: str) -> list[int]: + tokenizer = _get_tokenizer() + messages = [{"role": "user", "content": text}] + token_ids = normalize_token_ids(tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=False)) + assert len(token_ids) > _MIN_PROMPT_TOKENS, ( + f"Prompt too short ({len(token_ids)} tokens, need >{_MIN_PROMPT_TOKENS}). " + "The pipeline drops the first 34 chat-template prefix tokens; " + "use a longer prompt so content tokens remain after the drop." + ) + return token_ids + + +@pytest.fixture +def init_server(): + """Create and launch a vLLMOmniHttpServerLocal Ray actor with Qwen-Image.""" + model_path = _resolve_model_path(MODEL) + + ray.init( + runtime_env={ + "env_vars": { + "TOKENIZERS_PARALLELISM": "true", + "NCCL_DEBUG": "WARN", + "VLLM_LOGGING_LEVEL": "INFO", + } + }, + ignore_reinit_error=True, + ) + + rollout_cfg = OmegaConf.create( + { + "name": "vllm_omni", + "mode": "async", + "tensor_model_parallel_size": 1, + "data_parallel_size": 1, + "pipeline_model_parallel_size": 1, + "gpu_memory_utilization": 0.8, + "max_num_batched_tokens": 8192, + # QwenImagePipeline (and the logprob custom subclass) only supports + # serial request execution; max_num_seqs>1 enables request-level batching. + "max_num_seqs": 1, + "max_model_len": 1058, + "dtype": "bfloat16", + "load_format": "auto", + "enforce_eager": True, + "enable_chunked_prefill": False, + "enable_prefix_caching": False, + "enable_sleep_mode": False, + "free_cache_engine": True, + "disable_log_stats": True, + "logprobs_mode": "processed_logprobs", + "scheduling_policy": "fcfs", + "seed": 0, + "n": 4, + "pipeline": { + "height": 512, + "width": 512, + "num_inference_steps": 10, + }, + } + ) + + model_cfg = OmegaConf.create( + { + "path": str(model_path), + "local_path": str(model_path), + "tokenizer_path": TOKENIZER_MODEL, + "trust_remote_code": True, + "load_tokenizer": True, + } + ) + + ServerCls = ray.remote(vLLMOmniHttpServerLocal) + server = ServerCls.options( + runtime_env={ + "env_vars": { + "RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES": "1", + "RAY_EXPERIMENTAL_NOSET_ASCEND_RT_VISIBLE_DEVICES": "1", + "NCCL_CUMEM_ENABLE": "0", + } + }, + max_concurrency=10, + num_gpus=1, + ).remote( + config=rollout_cfg, + model_config=model_cfg, + rollout_mode=RolloutMode.STANDALONE, + workers=[], + replica_rank=0, + node_rank=0, + gpus_per_node=1, + nnodes=1, + cuda_visible_devices="0", + ) + + ray.get(server.launch_server.remote()) + + yield server + + ray.shutdown() + + +@pytest.mark.core_model +@pytest.mark.diffusion +@hardware_test(res={"cuda": "L4"}, num_cards=1) +def test_generate(init_server): + """Concurrent generate() calls covering basic output, logprobs, and multi-request correctness.""" + server = init_server + + prompts = [ + "a beautiful sunset over the ocean with vibrant orange and purple clouds " + "reflecting on the calm water surface near a rocky coastline", + "a fluffy orange cat sitting on a wooden windowsill looking outside at " + "a garden full of colorful flowers on a bright sunny afternoon", + "a majestic mountain landscape covered with fresh white snow under a " + "clear blue sky with pine trees in the foreground and a frozen lake", + "a futuristic city at night with neon lights glowing on tall glass " + "skyscrapers and flying vehicles soaring between the buildings", + ] + + refs = [] + for i, prompt in enumerate(prompts): + rid = f"test_{i}_{uuid4().hex[:8]}" + ref = server.generate.remote( + prompt_ids=_tokenize_prompt(prompt), + sampling_params={ + "num_inference_steps": 10, + "true_cfg_scale": 4.0, + "height": 512, + "width": 512, + "logprobs": i == 0, + }, + request_id=rid, + ) + refs.append(ref) + + results = ray.get(refs, timeout=600) + + for i, output in enumerate(results): + assert isinstance(output, DiffusionOutput), f"Request {i}: expected DiffusionOutput" + img = output.diffusion_output + assert isinstance(img, torch.Tensor) and img.ndim == 3, ( + f"Request {i}: expected CHW torch.Tensor, got {type(img).__name__} ndim={getattr(img, 'ndim', None)}" + ) + c, h, w = img.shape + assert c == 3, f"Request {i}: expected 3 channels (CHW), got {c}" + assert h > 0 and w > 0, f"Request {i}: image dimensions must be positive" + assert output.stop_reason in ("completed", "aborted", None), ( + f"Request {i}: unexpected stop_reason {output.stop_reason!r}" + ) + assert 0.0 <= float(img[0, 0, 0]) <= 1.0, f"Request {i}: pixel values must be in [0, 1]" + + print(f"All {len(prompts)} concurrent requests returned valid DiffusionOutput") diff --git a/tests/e2e/offline_inference/custom_pipeline/__init__.py b/tests/e2e/offline_inference/custom_pipeline/__init__.py deleted file mode 100644 index 99663a46a..000000000 --- a/tests/e2e/offline_inference/custom_pipeline/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from tests.e2e.offline_inference.custom_pipeline.qwen_image_pipeline_with_logprob import ( - QwenImagePipelineWithLogProbForTest, -) -from tests.e2e.offline_inference.custom_pipeline.worker_extension import ( - vLLMOmniColocateWorkerExtensionForTest, -) - -__all__ = [ - "QwenImagePipelineWithLogProbForTest", - "vLLMOmniColocateWorkerExtensionForTest", -] diff --git a/tests/e2e/offline_inference/custom_pipeline/flow_match_sde_scheduler.py b/tests/e2e/offline_inference/custom_pipeline/flow_match_sde_scheduler.py deleted file mode 100644 index f85eb6602..000000000 --- a/tests/e2e/offline_inference/custom_pipeline/flow_match_sde_scheduler.py +++ /dev/null @@ -1,220 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from __future__ import annotations - -import math -from dataclasses import dataclass -from typing import Literal - -import torch -from diffusers import FlowMatchEulerDiscreteScheduler -from diffusers.utils import BaseOutput -from diffusers.utils.torch_utils import randn_tensor - - -@dataclass -class FlowMatchSDEDiscreteSchedulerOutputForTest(BaseOutput): - """ - Output class for the scheduler's `step` function output. - - Args: - prev_sample (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_channels)` for images): - Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the - denoising loop. - log_prob (`torch.FloatTensor` of shape `(batch_size,)`, *optional*): - The log probability of the previous sample. - prev_sample_mean (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_channels)` for images): - The mean of the computed sample of previous timestep. - std_dev_t (`torch.FloatTensor` of shape `(batch_size, 1, 1)`): - The standard deviation used to compute `prev_sample`. - """ - - prev_sample: torch.FloatTensor - log_prob: torch.FloatTensor | None - prev_sample_mean: torch.FloatTensor - std_dev_t: torch.FloatTensor - - -class FlowMatchSDEDiscreteSchedulerForTest(FlowMatchEulerDiscreteScheduler): - """SDE version of the FlowMatchEulerDiscreteScheduler. - The implementation is based on FlowGRPO paper (https://arxiv.org/abs/2505.05470) - and diffusers v0.37 branch. - """ - - def step( - self, - model_output: torch.FloatTensor, - timestep: float | torch.FloatTensor, - sample: torch.FloatTensor, - s_churn: float = 0.0, - s_tmin: float = 0.0, - s_tmax: float = float("inf"), - s_noise: float = 1.0, - generator: torch.Generator | None = None, - per_token_timesteps: torch.Tensor | None = None, - return_dict: bool = True, - noise_level: float = 0.7, - prev_sample: torch.FloatTensor | None = None, - sde_type: Literal["sde", "cps"] = "sde", - return_logprobs: bool = True, - ) -> FlowMatchSDEDiscreteSchedulerOutputForTest | tuple: - """ - Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion - process from the learned model outputs (most often the predicted noise). - - Modified from https://github.com/yifan123/flow_grpo/blob/main/flow_grpo/diffusers_patch/sd3_sde_with_logprob.py - - Args: - model_output (`torch.FloatTensor`): - The direct output from learned diffusion model. - timestep (`float`): - The current discrete timestep in the diffusion chain. - sample (`torch.FloatTensor`): - A current instance of a sample created by the diffusion process. - s_churn (`float`): - s_tmin (`float`): - s_tmax (`float`): - s_noise (`float`, defaults to 1.0): - Scaling factor for noise added to the sample. - generator (`torch.Generator`, *optional*): - A random number generator. - per_token_timesteps (`torch.Tensor`, *optional*): - The timesteps for each token in the sample. - return_dict (`bool`): - Whether or not to return a - [`~schedulers.scheduling_flow_match_euler_discrete.FlowMatchSDEDiscreteSchedulerOutput`] or tuple. - noise_level (`float`, *optional*, defaults to 0.7): - The noise level used in the SDE. - prev_sample (`torch.FloatTensor`, *optional*): - The sample from the previous timestep. If not provided, it will be sampled inside the function. - sde_type (`str`, *optional*, defaults to "sde"): - The type of SDE to use. Choose between "sde" and "cps". - return_logprobs (`bool`, *optional*, defaults to True): - Whether to return log probabilities of the previous sample. - """ - - if isinstance(timestep, (int, torch.IntTensor, torch.LongTensor)): - raise ValueError( - ( - "Passing integer indices (e.g. from `enumerate(timesteps)`) as timesteps to" - " `FlowMatchEulerDiscreteScheduler.step()` is not supported. Make sure to pass" - " one of the `scheduler.timesteps` as a timestep." - ), - ) - - if self.step_index is None: - self._init_step_index(timestep) - - # Upcast to avoid precision issues when computing prev_sample - sample = sample.to(torch.float32) - if prev_sample is not None: - prev_sample = prev_sample.to(torch.float32) - - prev_sample, log_prob, prev_sample_mean, std_dev_t = self.sample_previous_step( - sample=sample, - model_output=model_output, - generator=generator, - per_token_timesteps=per_token_timesteps, - noise_level=noise_level, - prev_sample=prev_sample, - sde_type=sde_type, - return_logprobs=return_logprobs, - ) - - # upon completion increase step index by one - self._step_index += 1 - if per_token_timesteps is None: - # Cast sample back to model compatible dtype - prev_sample = prev_sample.to(model_output.dtype) - - if not return_dict: - return (prev_sample, log_prob, prev_sample_mean, std_dev_t) - - return FlowMatchSDEDiscreteSchedulerOutputForTest( - prev_sample=prev_sample, log_prob=log_prob, prev_sample_mean=prev_sample_mean, std_dev_t=std_dev_t - ) - - def sample_previous_step( - self, - sample: torch.Tensor, - model_output: torch.Tensor, - timestep: torch.FloatTensor | None = None, - generator: torch.Generator | None = None, - per_token_timesteps: torch.Tensor | None = None, - noise_level: float = 0.7, - prev_sample: torch.Tensor | None = None, - sde_type: Literal["cps", "sde"] = "sde", - return_logprobs: bool = True, - ): - assert sde_type in ["sde", "cps"] - assert sample.dtype == torch.float32 - if prev_sample is not None: - assert prev_sample.dtype == torch.float32 - - if per_token_timesteps is not None: - raise NotImplementedError("per_token_timesteps is not supported yet for FlowMatchSDEDiscreteScheduler.") - else: - if timestep is None: - sigma_idx = self.step_index - sigma = self.sigmas[sigma_idx] - sigma_prev = self.sigmas[sigma_idx + 1] - else: - sigma_idx = torch.tensor([self.index_for_timestep(t) for t in timestep]) - sigma = self.sigmas[sigma_idx].view(-1, *([1] * (len(sample.shape) - 1))) - sigma_prev = self.sigmas[sigma_idx + 1].view(-1, *([1] * (len(sample.shape) - 1))) - - sigma_max = self.sigmas[1] - dt = sigma_prev - sigma - - if sde_type == "sde": - std_dev_t = torch.sqrt(sigma / (1 - torch.where(sigma == 1, sigma_max, sigma))) * noise_level - - prev_sample_mean = ( - sample * (1 + std_dev_t**2 / (2 * sigma) * dt) - + model_output * (1 + std_dev_t**2 * (1 - sigma) / (2 * sigma)) * dt - ) - - if prev_sample is None: - variance_noise = randn_tensor( - model_output.shape, - generator=generator, - device=model_output.device, - dtype=model_output.dtype, - ) - prev_sample = prev_sample_mean + std_dev_t * torch.sqrt(-1 * dt) * variance_noise - - if return_logprobs: - log_prob = ( - -((prev_sample.detach() - prev_sample_mean) ** 2) / (2 * ((std_dev_t * torch.sqrt(-1 * dt)) ** 2)) - - torch.log(std_dev_t * torch.sqrt(-1 * dt)) - - torch.log(torch.sqrt(2 * torch.as_tensor(math.pi))) - ) - else: - log_prob = None - - elif sde_type == "cps": - std_dev_t = sigma_prev * math.sin(noise_level * math.pi / 2) - pred_original_sample = sample - sigma * model_output - noise_estimate = sample + model_output * (1 - sigma) - prev_sample_mean = pred_original_sample * (1 - sigma_prev) + noise_estimate * torch.sqrt( - sigma_prev**2 - std_dev_t**2 - ) - - if prev_sample is None: - variance_noise = randn_tensor( - model_output.shape, - generator=generator, - device=model_output.device, - dtype=model_output.dtype, - ) - prev_sample = prev_sample_mean + std_dev_t * variance_noise - - if return_logprobs: - log_prob = -((prev_sample.detach() - prev_sample_mean) ** 2) - else: - log_prob = None - - # mean along all but batch dimension - log_prob = log_prob.mean(dim=tuple(range(1, log_prob.ndim))) if log_prob is not None else None - return prev_sample, log_prob, prev_sample_mean, std_dev_t diff --git a/tests/e2e/offline_inference/custom_pipeline/worker_extension.py b/tests/e2e/offline_inference/custom_pipeline/worker_extension.py deleted file mode 100644 index 6ed29d5a8..000000000 --- a/tests/e2e/offline_inference/custom_pipeline/worker_extension.py +++ /dev/null @@ -1,77 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -"""Test worker extension that mirrors -``verl_omni.workers.rollout.vllm_rollout.utils.vLLMOmniColocateWorkerExtension`` - -Fidelity goals: - -* ``__new__`` calls ``set_death_signal()`` (Linux ``PR_SET_PDEATHSIG``) so - spawned vLLM workers die with the parent. -* ``__new__`` calls ``VLLMOmniHijackForTest.hijack()`` which monkey-patches - ``vllm_omni.diffusion.lora.manager.DiffusionLoRAManager._load_adapter`` - to accept in-memory LoRA tensors (verbatim port of - ``verl_omni.utils.vllm_omni.utils.VLLMOmniHijack.hijack``). - -The ``verl.utils.vllm.VLLMHijack.hijack()`` call (which patches vLLM's -``LRUCacheWorkerLoRAManager._load_adapter`` for AR/text mode) is -intentionally skipped because (a) ``verl`` is not importable from this -repo and (b) the diffusion test path never reaches that manager. -""" - -from __future__ import annotations - -import ctypes -import logging -import os -import platform -import signal - -from msgspec import field - -from vllm_omni.diffusion.worker.diffusion_worker import CustomPipelineWorkerExtension -from vllm_omni.lora.request import LoRARequest as OmniLoRARequest - -logger = logging.getLogger(__name__) - - -class OmniTensorLoRARequestForTest(OmniLoRARequest): - peft_config: dict = field(default=None) - lora_tensors: dict = field(default=None) - - -def set_death_signal() -> None: - """Verbatim port of ``verl.workers.rollout.vllm_rollout.utils.set_death_signal``.""" - if platform.system() != "Linux": - return - try: - libc = ctypes.CDLL("libc.so.6") - libc.prctl(1, signal.SIGKILL) - if os.getppid() == 1: - os.kill(os.getpid(), signal.SIGKILL) - except Exception: # noqa: BLE001 - # libc.so.6 may not be present (musl, alpine) — best-effort only. - pass - - -# --------------------------------------------------------------------- -# The worker extension itself -# --------------------------------------------------------------------- - - -class vLLMOmniColocateWorkerExtensionForTest(CustomPipelineWorkerExtension): - """Mirror of ``vLLMOmniColocateWorkerExtension`` (verl-omni). - - The production ``__new__`` runs ``set_death_signal`` + ``VLLMOmniHijack.hijack`` - on every vLLM worker process. Replicating both ensures this test - reproduces the same monkey-patched environment. - """ - - def __new__(cls, **kwargs): - set_death_signal() - return super().__new__(cls) - - @staticmethod - def test_extension_name() -> str: - """Return a stable identifier for assertions in unit tests.""" - return "vllm-omni-colocate-worker-extension-for-test" diff --git a/tests/e2e/offline_inference/test_soulxsinger.py b/tests/e2e/offline_inference/test_soulxsinger_expansion.py similarity index 99% rename from tests/e2e/offline_inference/test_soulxsinger.py rename to tests/e2e/offline_inference/test_soulxsinger_expansion.py index 465f5547b..c3a6091d6 100644 --- a/tests/e2e/offline_inference/test_soulxsinger.py +++ b/tests/e2e/offline_inference/test_soulxsinger_expansion.py @@ -38,7 +38,7 @@ if not PROMPT_AUDIO.is_file() or not TARGET_AUDIO.is_file(): allow_module_level=True, ) -pytestmark = [pytest.mark.advanced_model, pytest.mark.diffusion, pytest.mark.tts] +pytestmark = [pytest.mark.slow, pytest.mark.diffusion, pytest.mark.tts] _CASES = ( pytest.param( diff --git a/tests/e2e/offline_inference/test_step_audio2_expansion.py b/tests/e2e/offline_inference/test_step_audio2_expansion.py index 362f2f33f..00fc7bbe2 100644 --- a/tests/e2e/offline_inference/test_step_audio2_expansion.py +++ b/tests/e2e/offline_inference/test_step_audio2_expansion.py @@ -6,7 +6,6 @@ E2E expansion tests for Step-Audio2 offline inference (nightly CI). Full-model inference with audio input and audio output. """ -from pathlib import Path from typing import Any import numpy as np @@ -15,10 +14,11 @@ import pytest from tests.helpers.mark import hardware_test from tests.helpers.media import generate_synthetic_audio from tests.helpers.runtime import OmniRunner +from tests.helpers.stage_config import get_deploy_config_path from vllm_omni.outputs import OmniRequestOutput MODEL = "stepfun-ai/Step-Audio-2-mini" -STAGE_CONFIG = str(Path(__file__).parent / "stage_configs" / "step_audio2_ci.yaml") +STAGE_CONFIG = get_deploy_config_path("step_audio2_thinker_ci.yaml") TEST_PARAMS = [(MODEL, STAGE_CONFIG)] pytestmark = [pytest.mark.slow, pytest.mark.tts] diff --git a/tests/e2e/online_serving/test_dreamzero_expansion.py b/tests/e2e/online_serving/test_dreamzero_expansion.py index 38e7d4b16..66d558120 100644 --- a/tests/e2e/online_serving/test_dreamzero_expansion.py +++ b/tests/e2e/online_serving/test_dreamzero_expansion.py @@ -1,34 +1,37 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""E2E online serving test for DreamZero OpenPI websocket serving.""" +"""E2E online serving tests for DreamZero OpenPI websocket serving.""" from __future__ import annotations import os from pathlib import Path +import cv2 import numpy as np import pytest -from tests.dreamzero import openpi_client_helper as openpi_client from tests.helpers.mark import hardware_test -from tests.helpers.runtime import OmniServerParams, get_open_port +from tests.helpers.runtime import ( + DREAMZERO_ACTION_DIM, + DREAMZERO_ACTION_HORIZON, + DREAMZERO_CAMERA_FILES, + OmniServerParams, + OpenPIWebSocketResponse, + get_open_port, +) +from tests.helpers.stage_config import get_deploy_config_path os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" MODEL = "GEAR-Dreams/DreamZero-DROID" - test_params = [ OmniServerParams( model=MODEL, - port=8091, - server_args=[ - "--deploy-config", - "vllm_omni/deploy/dreamzero_tp1_cfg2.yaml", - ], + stage_config_path=get_deploy_config_path("dreamzero_tp1_cfg2.yaml"), env_dict={ "ATTENTION_BACKEND": "torch", "DIFFUSION_ATTENTION_BACKEND": "TORCH_SDPA", @@ -39,10 +42,10 @@ test_params = [ ] -def _write_synthetic_video(path: Path, cv2_module, *, channel: int) -> None: +def _write_synthetic_video(path: Path, *, channel: int) -> None: path.parent.mkdir(parents=True, exist_ok=True) height, width, num_frames = 180, 320, 24 - writer = cv2_module.VideoWriter(str(path), cv2_module.VideoWriter_fourcc(*"mp4v"), 15.0, (width, height)) + writer = cv2.VideoWriter(str(path), cv2.VideoWriter_fourcc(*"mp4v"), 15.0, (width, height)) if not writer.isOpened(): raise RuntimeError(f"Failed to open video writer for {path}") try: @@ -50,39 +53,47 @@ def _write_synthetic_video(path: Path, cv2_module, *, channel: int) -> None: frame = np.zeros((height, width, 3), dtype=np.uint8) frame[..., channel] = (frame_idx * 7) % 255 frame[..., (channel + 1) % 3] = 64 - writer.write(cv2_module.cvtColor(frame, cv2_module.COLOR_RGB2BGR)) + writer.write(cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)) finally: writer.release() -def _write_synthetic_dreamzero_videos(client_mod, video_dir: Path) -> None: - for channel, file_name in enumerate(client_mod.CAMERA_FILES.values()): - _write_synthetic_video(video_dir / file_name, client_mod.cv2, channel=channel) +def _write_synthetic_dreamzero_videos(video_dir: Path) -> None: + for channel, file_name in enumerate(DREAMZERO_CAMERA_FILES.values()): + _write_synthetic_video(video_dir / file_name, channel=channel) + + +def _validate_dreamzero_openpi_session(response: OpenPIWebSocketResponse) -> None: + metadata = response.server_metadata + assert tuple(metadata["image_resolution"]) == (180, 320) + assert metadata["n_external_cameras"] == 2 + assert metadata["needs_wrist_camera"] is True + assert metadata["needs_stereo_camera"] is False + assert metadata["needs_session_id"] is True + assert metadata["action_space"] == "joint_position" + assert response.operation_responses[-1]["status"] == "reset successful" + + action_tensors = response.action_tensors + assert action_tensors is not None + assert len(action_tensors) == 3 + for index, action in enumerate(action_tensors): + assert action.shape == (DREAMZERO_ACTION_HORIZON, DREAMZERO_ACTION_DIM) + assert action.dtype == np.float32 + assert np.isfinite(action).all() @pytest.mark.slow @pytest.mark.diffusion -@pytest.mark.distributed_cuda @hardware_test(res={"cuda": "H100"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) -def test_dreamzero_openpi_online(omni_server, tmp_path: Path) -> None: - try: - openpi_client.require_dependencies() - except ModuleNotFoundError as exc: - pytest.skip(str(exc)) - +def test_dreamzero_openpi_online(omni_server, openai_client, tmp_path: Path) -> None: video_dir = tmp_path / "dreamzero_videos" - _write_synthetic_dreamzero_videos(openpi_client, video_dir) - result = openpi_client.run_policy_session( - host=omni_server.host, - port=omni_server.port, - video_dir=video_dir, - session_id="dreamzero-online-e2e", - ) - - openpi_client.validate_session_result(result) - - metadata = result["metadata"] - assert metadata["needs_session_id"] is True - assert metadata["needs_stereo_camera"] is False - assert tuple(metadata["image_resolution"]) == (180, 320) + _write_synthetic_dreamzero_videos(video_dir) + response = openai_client.send_robot_openpi_ws_request( + { + "run_dreamzero_policy_session": True, + "video_dir": video_dir, + "session_id": "dreamzero-online-e2e", + } + )[0] + _validate_dreamzero_openpi_session(response) diff --git a/tests/e2e/online_serving/test_gr00t_openpi.py b/tests/e2e/online_serving/test_gr00t_openpi_expansion.py similarity index 66% rename from tests/e2e/online_serving/test_gr00t_openpi.py rename to tests/e2e/online_serving/test_gr00t_openpi_expansion.py index 40b8293d6..70e11088d 100644 --- a/tests/e2e/online_serving/test_gr00t_openpi.py +++ b/tests/e2e/online_serving/test_gr00t_openpi_expansion.py @@ -7,14 +7,19 @@ import os import numpy as np import pytest -from tests.gr00t import openpi_client_helper as openpi_client from tests.helpers.mark import hardware_test -from tests.helpers.runtime import OmniServerParams +from tests.helpers.runtime import ( + OmniServerParams, + OpenPIWebSocketResponse, + build_openpi_droid_observation, +) from tests.helpers.stage_config import get_deploy_config_path os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" MODEL = "nvidia/GR00T-N1.7-3B" +ACTION_KEYS = frozenset({"eef_9d", "gripper_position", "joint_position"}) +EXPECTED_EMBODIMENT_TAG = "OXE_DROID_RELATIVE_EEF_RELATIVE_JOINT" pytest.importorskip("websockets") pytest.importorskip("openpi_client.msgpack_numpy") @@ -34,7 +39,7 @@ test_params = [ ] # Reference values captured from the Isaac-GR00T ZMQ server (nvidia/GR00T-N1.7-3B, -# OXE_DROID_RELATIVE_EEF_RELATIVE_JOINT embodiment) using build_droid_observation() +# OXE_DROID_RELATIVE_EEF_RELATIVE_JOINT embodiment) using build_openpi_droid_observation() # inputs: zero images (256×256), identity eef_9d, zero gripper/joint, "pick up the object". # Outputs are bit-reproducible across resets and across runs (max_diff=0.0). _REF_EEF_9D = np.array( @@ -112,32 +117,65 @@ _REF_JOINT = np.array( ) # rows = step 0, step 4, step 39 -@pytest.mark.full_model +def _validate_gr00t_openpi_session(response: OpenPIWebSocketResponse) -> None: + metadata = response.server_metadata + for key in ("action_horizon", "action_keys", "embodiment_tag", "needs_session_id"): + assert key in metadata, f"Missing GR00T metadata key: {key}" + + assert metadata["embodiment_tag"] == EXPECTED_EMBODIMENT_TAG + assert metadata["needs_session_id"] is True + assert frozenset(metadata["action_keys"]) == ACTION_KEYS + assert response.operation_responses[-1]["status"] == "reset successful" + + actions = response.actions + assert actions is not None + assert frozenset(actions) == ACTION_KEYS + + action_horizon = int(metadata["action_horizon"]) + expected_shapes = { + "eef_9d": (1, action_horizon, 9), + "gripper_position": (1, action_horizon, 1), + "joint_position": (1, action_horizon, 7), + } + for key, expected_shape in expected_shapes.items(): + assert actions[key].shape == expected_shape + assert actions[key].dtype == np.float32 + assert np.isfinite(actions[key]).all() + + +@pytest.mark.slow @pytest.mark.diffusion @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) -def test_gr00t_n1d7_openpi_online(omni_server) -> None: - result = openpi_client.run_policy_session( - host=omni_server.host, - port=omni_server.port, - session_id="gr00t-online-e2e", - ) - openpi_client.validate_session_result(result) +def test_gr00t_n1d7_openpi_online(omni_server, openai_client) -> None: + response = openai_client.send_robot_openpi_ws_request( + { + "run_default_policy_session": True, + "session_id": "gr00t-online-e2e", + } + )[0] + _validate_gr00t_openpi_session(response) -@pytest.mark.full_model +@pytest.mark.slow @pytest.mark.diffusion @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) -def test_gr00t_n1d7_openpi_precision(omni_server) -> None: +def test_gr00t_n1d7_openpi_precision(omni_server, openai_client) -> None: """Assert actions match Isaac-GR00T reference (GR00T_NOISE_SEED=42, zero inputs).""" - client = openpi_client.OpenPIWebsocketClient(host=omni_server.host, port=omni_server.port) - try: - obs = openpi_client.build_droid_observation(session_id="gr00t-precision-e2e") - client.reset({}) - actions = client.infer(obs) - finally: - client.close() + response = openai_client.send_robot_openpi_ws_request( + { + "operations": [ + {"endpoint": "reset", "payload": {}}, + { + "endpoint": "infer", + "payload": build_openpi_droid_observation(session_id="gr00t-precision-e2e"), + }, + ], + } + )[0] + actions = response.actions + assert actions is not None steps = [0, 4, 39] np.testing.assert_allclose( diff --git a/tests/e2e/online_serving/test_omnivoice_expansion.py b/tests/e2e/online_serving/test_omnivoice_expansion.py index ff41ee413..ab35b74ea 100644 --- a/tests/e2e/online_serving/test_omnivoice_expansion.py +++ b/tests/e2e/online_serving/test_omnivoice_expansion.py @@ -25,7 +25,7 @@ try: except ImportError: _HAS_VOICE_CLONE = False -pytestmark = [pytest.mark.full_model, pytest.mark.tts] +pytestmark = [pytest.mark.slow, pytest.mark.tts] MODEL = "k2-fsa/OmniVoice" diff --git a/tests/test_arg_utils.py b/tests/engine/test_arg_utils_shared_fields.py similarity index 100% rename from tests/test_arg_utils.py rename to tests/engine/test_arg_utils_shared_fields.py diff --git a/tests/test_data_entry_keys.py b/tests/engine/test_data_entry_keys.py similarity index 100% rename from tests/test_data_entry_keys.py rename to tests/engine/test_data_entry_keys.py diff --git a/tests/test_omni_request.py b/tests/engine/test_omni_request.py similarity index 100% rename from tests/test_omni_request.py rename to tests/engine/test_omni_request.py diff --git a/tests/test_outputs.py b/tests/engine/test_omni_request_output.py similarity index 100% rename from tests/test_outputs.py rename to tests/engine/test_omni_request_output.py diff --git a/tests/engine/test_orchestrator_error_handling.py b/tests/engine/test_orchestrator_error_handling.py index 0553c03ff..ed32732ba 100644 --- a/tests/engine/test_orchestrator_error_handling.py +++ b/tests/engine/test_orchestrator_error_handling.py @@ -73,9 +73,17 @@ def orchestrator_factory(): class FakeDeadLLMStageClient(FakeStageClient): - """LLM stage client that raises EngineDeadError on get_output_async.""" + """LLM stage client that raises EngineDeadError once a request is in-flight. + + Raising on the first poll races with ``add_request``: under suite load the + orchestrator can die and drain before the test enqueues anything, so no + fatal ErrorMessage is ever published. Keep returning empty outputs until + ``add_request_async`` has been called, then fail on the next poll. + """ async def get_output_async(self): + if not self.add_request_calls: + return SimpleNamespace(outputs=[]) raise EngineDeadError("Stage-0 engine core is dead") diff --git a/tests/test_speaker_metadata_persistence.py b/tests/entrypoints/openai_api/test_speaker_metadata_persistence.py similarity index 100% rename from tests/test_speaker_metadata_persistence.py rename to tests/entrypoints/openai_api/test_speaker_metadata_persistence.py diff --git a/tests/entrypoints/openai_api/test_speech_usage.py b/tests/entrypoints/openai_api/test_speech_usage.py index 8211dab88..2fba10516 100644 --- a/tests/entrypoints/openai_api/test_speech_usage.py +++ b/tests/entrypoints/openai_api/test_speech_usage.py @@ -16,6 +16,8 @@ from vllm_omni.entrypoints.openai.speech_usage import ( qwen3_tts_input_token_details, ) +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + # A deterministic stand-in for a real tokenizer: 1 token per whitespace word. def word_count(text: str) -> int: diff --git a/tests/entrypoints/openai_api/test_stage_params.py b/tests/entrypoints/openai_api/test_stage_params.py index f9fd40349..5db768b30 100644 --- a/tests/entrypoints/openai_api/test_stage_params.py +++ b/tests/entrypoints/openai_api/test_stage_params.py @@ -3,6 +3,7 @@ from types import SimpleNamespace +import pytest from vllm import SamplingParams from vllm_omni.entrypoints.openai.stage_params import ( @@ -12,6 +13,8 @@ from vllm_omni.entrypoints.openai.stage_params import ( ) from vllm_omni.inputs.data import OmniDiffusionSamplingParams +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + def test_resolve_stage_sampling_params_clones_stage_default(): default = SamplingParams(temperature=0.2, seed=11) diff --git a/tests/entrypoints/openai_api/test_tts_adapter.py b/tests/entrypoints/openai_api/test_tts_adapter.py index 2930a204d..5f6d5930a 100644 --- a/tests/entrypoints/openai_api/test_tts_adapter.py +++ b/tests/entrypoints/openai_api/test_tts_adapter.py @@ -15,6 +15,9 @@ from vllm_omni.entrypoints.openai.tts_adapters import ( ) from vllm_omni.entrypoints.openai.tts_adapters.qwen3_tts import Qwen3TTSAdapter +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + # Every dedicated TTS model-type must have an adapter so the orchestrator's # uniform ``self._adapter.build(...)`` dispatch covers it. EXPECTED_MODEL_TYPES = { diff --git a/tests/entrypoints/openai_api/test_video_frame_filter.py b/tests/entrypoints/openai_api/test_video_frame_filter.py index 70df6e34c..ecada7a55 100644 --- a/tests/entrypoints/openai_api/test_video_frame_filter.py +++ b/tests/entrypoints/openai_api/test_video_frame_filter.py @@ -12,6 +12,8 @@ from tests.entrypoints.openai_api.conftest_video import ( ) from vllm_omni.entrypoints.openai.video_frame_filter import FrameSimilarityFilter +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + class TestFrameSimilarityFilter: def test_first_frame_always_retained(self): diff --git a/tests/entrypoints/openai_api/test_video_stream_handler.py b/tests/entrypoints/openai_api/test_video_stream_handler.py index a5ce3d616..e382b5e32 100644 --- a/tests/entrypoints/openai_api/test_video_stream_handler.py +++ b/tests/entrypoints/openai_api/test_video_stream_handler.py @@ -22,6 +22,9 @@ from vllm_omni.entrypoints.openai.video_stream_session import ( VideoStreamHandler, ) +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- diff --git a/tests/entrypoints/openai_api/test_video_stream_session.py b/tests/entrypoints/openai_api/test_video_stream_session.py index b0a12b1e4..b91723c26 100644 --- a/tests/entrypoints/openai_api/test_video_stream_session.py +++ b/tests/entrypoints/openai_api/test_video_stream_session.py @@ -15,6 +15,9 @@ from vllm_omni.entrypoints.openai.video_stream_session import ( VideoStreamSession, ) +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + # --------------------------------------------------------------------------- # VideoStreamConfig # --------------------------------------------------------------------------- diff --git a/tests/entrypoints/test_pd_disaggregation.py b/tests/entrypoints/test_pd_disaggregation.py index 774e6d1a8..c896ccb88 100644 --- a/tests/entrypoints/test_pd_disaggregation.py +++ b/tests/entrypoints/test_pd_disaggregation.py @@ -22,6 +22,9 @@ from vllm import SamplingParams from vllm_omni.entrypoints.pd_utils import PDDisaggregationMixin +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + pytestmark = pytest.mark.skip(reason="Temporarily skip PD entrypoint tests while PD config is being removed.") # Suppress noisy DeprecationWarnings from optional Swig bindings imported by vLLM dependencies. diff --git a/tests/entrypoints/test_stage_utils.py b/tests/entrypoints/test_stage_utils.py index c4c0667df..e2890797f 100644 --- a/tests/entrypoints/test_stage_utils.py +++ b/tests/entrypoints/test_stage_utils.py @@ -6,6 +6,8 @@ from pytest_mock import MockerFixture from vllm_omni.entrypoints.stage_utils import _map_device_list, resolve_stage_physical_devices, set_stage_devices +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + def _make_dummy_torch(call_log): class _Props: diff --git a/tests/gr00t/__init__.py b/tests/gr00t/__init__.py deleted file mode 100644 index 208f01a7c..000000000 --- a/tests/gr00t/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/tests/gr00t/openpi_client_helper.py b/tests/gr00t/openpi_client_helper.py deleted file mode 100644 index e5943ca33..000000000 --- a/tests/gr00t/openpi_client_helper.py +++ /dev/null @@ -1,183 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from dataclasses import dataclass -from typing import Any - -import numpy as np - -try: - import websockets.sync.client as websockets_client -except ImportError: # pragma: no cover - optional e2e dependency - websockets_client = None - -try: - from openpi_client import msgpack_numpy -except ImportError: # pragma: no cover - optional e2e dependency - msgpack_numpy = None - -PING_INTERVAL_SECS = 300 -PING_TIMEOUT_SECS = 3600 -DEFAULT_HOST = "127.0.0.1" -DEFAULT_PORT = 8000 -DEFAULT_PATH = "/v1/realtime/robot/openpi" -DEFAULT_SESSION_ID = "gr00t-smoke" -ACTION_KEYS = {"eef_9d", "gripper_position", "joint_position"} -LANGUAGE_KEY = "annotation.language.language_instruction" - - -def _identity_eef_9d_state() -> np.ndarray: - state = np.zeros((1, 1, 9), dtype=np.float32) - state[..., 3:] = np.array([1, 0, 0, 0, 1, 0], dtype=np.float32) - return state - - -def require_dependencies() -> None: - missing = [] - if websockets_client is None: - missing.append("websockets") - if msgpack_numpy is None: - missing.append("openpi-client") - if missing: - raise ModuleNotFoundError(f"GR00T OpenPI test dependencies are missing: {', '.join(missing)}") - - -@dataclass(frozen=True) -class Gr00tServerMetadata: - action_horizon: int - action_keys: set[str] - embodiment_tag: str - needs_session_id: bool - - @classmethod - def from_dict(cls, payload: dict[str, Any]): - required_keys = ("action_horizon", "action_keys", "embodiment_tag", "needs_session_id") - missing_keys = [key for key in required_keys if key not in payload] - if missing_keys: - raise ValueError(f"Missing GR00T metadata keys: {missing_keys}") - - return cls( - action_horizon=int(payload["action_horizon"]), - action_keys={str(key) for key in payload["action_keys"]}, - embodiment_tag=str(payload["embodiment_tag"]), - needs_session_id=bool(payload["needs_session_id"]), - ) - - -class OpenPIWebsocketClient: - def __init__( - self, - *, - host: str = DEFAULT_HOST, - port: int = DEFAULT_PORT, - path: str = DEFAULT_PATH, - ) -> None: - require_dependencies() - self._uri = f"ws://{host}:{port}{path}" - self._packer = msgpack_numpy.Packer() - self._ws, self._server_metadata = self._connect() - - def _connect(self): - conn = websockets_client.connect( - self._uri, - compression=None, - max_size=None, - ping_interval=PING_INTERVAL_SECS, - ping_timeout=PING_TIMEOUT_SECS, - ) - metadata = msgpack_numpy.unpackb(conn.recv()) - if not isinstance(metadata, dict): - raise TypeError(f"Expected dict metadata from server, got {type(metadata)!r}") - return conn, metadata - - def get_server_metadata(self) -> dict[str, Any]: - return dict(self._server_metadata) - - def infer(self, obs: dict[str, Any]) -> dict[str, np.ndarray]: - payload = dict(obs) - payload["endpoint"] = "infer" - self._ws.send(self._packer.pack(payload)) - response = msgpack_numpy.unpackb(self._ws.recv()) - if isinstance(response, dict) and response.get("type") == "error": - raise RuntimeError(f"Inference failed: {response['message']}") - if not isinstance(response, dict): - raise RuntimeError(f"Expected dict actions from GR00T OpenPI endpoint, got {type(response)!r}") - return {str(key): np.asarray(value, dtype=np.float32) for key, value in response.items()} - - def reset(self, reset_info: dict[str, Any] | None = None) -> str: - payload = dict(reset_info or {}) - payload["endpoint"] = "reset" - self._ws.send(self._packer.pack(payload)) - response = msgpack_numpy.unpackb(self._ws.recv()) - if not isinstance(response, dict) or response.get("status") != "reset successful": - raise RuntimeError(f"Unexpected reset response: {response!r}") - return str(response["status"]) - - def close(self) -> None: - self._ws.close() - - -def build_droid_observation(*, session_id: str = DEFAULT_SESSION_ID) -> dict[str, Any]: - return { - "session_id": session_id, - "video": { - "exterior_image_1_left": np.zeros((1, 2, 256, 256, 3), dtype=np.uint8), - "wrist_image_left": np.zeros((1, 2, 256, 256, 3), dtype=np.uint8), - }, - "state": { - "eef_9d": _identity_eef_9d_state(), - "gripper_position": np.zeros((1, 1, 1), dtype=np.float32), - "joint_position": np.zeros((1, 1, 7), dtype=np.float32), - }, - "language": {LANGUAGE_KEY: [["pick up the object"]]}, - } - - -def run_policy_session( - *, - host: str = DEFAULT_HOST, - port: int = DEFAULT_PORT, - path: str = DEFAULT_PATH, - session_id: str = DEFAULT_SESSION_ID, -) -> dict[str, Any]: - client = OpenPIWebsocketClient(host=host, port=port, path=path) - try: - metadata = client.get_server_metadata() - actions = client.infer(build_droid_observation(session_id=session_id)) - reset_status = client.reset({}) - return { - "metadata": metadata, - "actions": actions, - "reset_status": reset_status, - "session_id": session_id, - } - finally: - client.close() - - -def validate_session_result(result: dict[str, Any]) -> None: - metadata = Gr00tServerMetadata.from_dict(result["metadata"]) - if metadata.embodiment_tag != "OXE_DROID_RELATIVE_EEF_RELATIVE_JOINT": - raise AssertionError(f"Unexpected embodiment_tag: {metadata.embodiment_tag}") - if not metadata.needs_session_id: - raise AssertionError("GR00T test expects needs_session_id metadata") - if metadata.action_keys != ACTION_KEYS: - raise AssertionError(f"Unexpected action keys: {metadata.action_keys}") - if result["reset_status"] != "reset successful": - raise AssertionError(f"Unexpected reset status: {result['reset_status']!r}") - - actions = result["actions"] - if set(actions) != ACTION_KEYS: - raise AssertionError(f"Unexpected action keys: {set(actions)}") - expected_shapes = { - "eef_9d": (1, metadata.action_horizon, 9), - "gripper_position": (1, metadata.action_horizon, 1), - "joint_position": (1, metadata.action_horizon, 7), - } - for key, expected_shape in expected_shapes.items(): - if actions[key].shape != expected_shape: - raise AssertionError(f"Action {key} shape mismatch: expected {expected_shape}, got {actions[key].shape}") - if actions[key].dtype != np.float32: - raise AssertionError(f"Action {key} dtype mismatch: expected float32, got {actions[key].dtype}") - if not np.isfinite(actions[key]).all(): - raise AssertionError(f"Action {key} contains non-finite values") diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py index 04d51fb90..9929beab8 100644 --- a/tests/helpers/__init__.py +++ b/tests/helpers/__init__.py @@ -1,10 +1,11 @@ """Shared, importable test helper utilities. Submodules (``assertions``, ``env``, ``media``, ``runtime``, …) are imported -explicitly by callers. Avoid star-importing everything here: that ran before -refactor only inside the old monolithic ``conftest``; a greedy ``__init__`` -changes import order and can affect in-process Omni (``OmniRunner`` / offline -e2e) vs subprocess-based ``OmniServer`` tests. +explicitly by callers. Unit tests for these helpers live under ``tests/``. +Avoid star-importing everything here: that ran before refactor only inside the +old monolithic ``conftest``; a greedy ``__init__`` changes import order and can +affect in-process Omni (``OmniRunner`` / offline e2e) vs subprocess-based +``OmniServer`` tests. """ import pytest diff --git a/tests/helpers/env.py b/tests/helpers/env.py index dcf2484cd..39cec30e7 100644 --- a/tests/helpers/env.py +++ b/tests/helpers/env.py @@ -27,6 +27,32 @@ def get_physical_device_indices(devices): return [index_mapping[i] for i in devices if i in index_mapping] +def pick_least_used_device_indices(num_devices: int) -> list[int]: + """Return physical device indices for the least-used accelerators. + + Args: + num_devices: Number of device indices to return. + + Returns: + Physical device indices sorted from least to most used memory. + """ + if num_devices <= 0: + raise ValueError(f"num_devices must be positive, got {num_devices}.") + + logical_count = current_omni_platform.get_device_count() + if logical_count < num_devices: + raise RuntimeError(f"Need {num_devices} devices, but only {logical_count} are available.") + + device_usage: list[tuple[int, int]] = [] + for device_index in range(logical_count): + with current_omni_platform.device(device_index): + free_bytes, total_bytes = current_omni_platform.get_device_memory() + device_usage.append((total_bytes - free_bytes, device_index)) + device_usage.sort() + logical_indices = [device_index for _, device_index in device_usage[:num_devices]] + return get_physical_device_indices(logical_indices) + + def wait_for_gpu_memory_to_clear( *, devices: list[int], @@ -264,6 +290,7 @@ class DeviceMemoryMonitor: __all__ = [ "DeviceMemoryMonitor", "get_physical_device_indices", + "pick_least_used_device_indices", "run_post_test_cleanup", "run_pre_test_cleanup", "wait_for_gpu_memory_to_clear", diff --git a/tests/helpers/runtime.py b/tests/helpers/runtime.py index 39baf1ce0..f8a5416e4 100644 --- a/tests/helpers/runtime.py +++ b/tests/helpers/runtime.py @@ -22,6 +22,7 @@ from pathlib import Path from typing import Any, NamedTuple, cast from urllib.parse import quote +import numpy as np import psutil import requests import soundfile as sf @@ -802,6 +803,189 @@ class WebSocketJsonResponse: json_body: dict[str, Any] +@dataclass +class OpenPIWebSocketResponse: + """Msgpack WebSocket session against ``/v1/realtime/robot/openpi``.""" + + server_metadata: dict[str, Any] + operation_responses: list[Any] + actions: dict[str, np.ndarray] | None = None + action_tensors: list[np.ndarray] | None = None + + +def build_openpi_droid_observation(*, session_id: str = "gr00t-smoke") -> dict[str, Any]: + """Build a minimal DROID-style observation payload for the OpenPI robot endpoint.""" + identity_eef_9d = np.zeros((1, 1, 9), dtype=np.float32) + identity_eef_9d[..., 3:] = np.array([1, 0, 0, 0, 1, 0], dtype=np.float32) + return { + "session_id": session_id, + "video": { + "exterior_image_1_left": np.zeros((1, 2, 256, 256, 3), dtype=np.uint8), + "wrist_image_left": np.zeros((1, 2, 256, 256, 3), dtype=np.uint8), + }, + "state": { + "eef_9d": identity_eef_9d, + "gripper_position": np.zeros((1, 1, 1), dtype=np.float32), + "joint_position": np.zeros((1, 1, 7), dtype=np.float32), + }, + "language": {"annotation.language.language_instruction": [["pick up the object"]]}, + } + + +DREAMZERO_DEFAULT_PROMPT = ( + "Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan" +) +DREAMZERO_ACTION_HORIZON = 24 +DREAMZERO_ACTION_DIM = 8 +DREAMZERO_CAMERA_FILES = { + "observation/exterior_image_0_left": "exterior_image_1_left.mp4", + "observation/exterior_image_1_left": "exterior_image_2_left.mp4", + "observation/wrist_image_left": "wrist_image_left.mp4", +} + + +def _require_opencv() -> Any: + try: + import cv2 + except ImportError as exc: # pragma: no cover - optional e2e dependency + raise ModuleNotFoundError("DreamZero OpenPI test dependencies are missing: opencv-python") from exc + return cv2 + + +def load_dreamzero_camera_frames(video_dir: Path) -> dict[str, np.ndarray]: + cv2 = _require_opencv() + camera_frames: dict[str, np.ndarray] = {} + for camera_key, file_name in DREAMZERO_CAMERA_FILES.items(): + video_path = video_dir / file_name + if not video_path.exists(): + raise FileNotFoundError(f"Missing DreamZero test asset: {video_path}") + cap = cv2.VideoCapture(str(video_path)) + frames = [] + while True: + ok, frame = cap.read() + if not ok: + break + frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) + cap.release() + if not frames: + raise RuntimeError(f"No frames loaded from {video_path}") + camera_frames[camera_key] = np.stack(frames, axis=0) + return camera_frames + + +def build_dreamzero_demo_observations( + camera_frames: dict[str, np.ndarray], + *, + prompt: str, + session_id: str, + num_chunks: int = 2, +) -> list[dict[str, Any]]: + if num_chunks < 1: + raise ValueError("num_chunks must be at least 1") + + relative_offsets = [-23, -16, -8, 0] + total_frames = min(frames.shape[0] for frames in camera_frames.values()) + frame_schedules = [[0]] + current_frame = 23 + for _ in range(num_chunks - 1): + indices = [max(current_frame + offset, 0) for offset in relative_offsets] + if indices[-1] >= total_frames: + break + frame_schedules.append(indices) + current_frame += DREAMZERO_ACTION_HORIZON + + observations: list[dict[str, Any]] = [] + for frame_indices in frame_schedules: + obs: dict[str, Any] = {} + for camera_key, all_frames in camera_frames.items(): + selected = all_frames[frame_indices] + obs[camera_key] = selected[0] if len(frame_indices) == 1 else selected + obs["observation/joint_position"] = np.zeros(7, dtype=np.float32) + obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32) + obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32) + obs["prompt"] = prompt + obs["session_id"] = session_id + observations.append(obs) + return observations + + +class OpenPIWebSocketSession: + """Persistent msgpack session for ``/v1/realtime/robot/openpi``.""" + + DEFAULT_PING_INTERVAL_SECS = 300 + DEFAULT_PING_TIMEOUT_SECS = 3600 + + @staticmethod + def _require_dependencies() -> tuple[Any, Any]: + try: + import websockets.sync.client as websockets_client + except ImportError as exc: # pragma: no cover - optional e2e dependency + raise ModuleNotFoundError("GR00T OpenPI test dependencies are missing: websockets") from exc + try: + from openpi_client import msgpack_numpy + except ImportError as exc: # pragma: no cover - optional e2e dependency + raise ModuleNotFoundError("GR00T OpenPI test dependencies are missing: openpi-client") from exc + return websockets_client, msgpack_numpy + + def __init__( + self, + uri: str, + *, + ping_interval: float = DEFAULT_PING_INTERVAL_SECS, + ping_timeout: float = DEFAULT_PING_TIMEOUT_SECS, + open_timeout: float = 120.0, + close_timeout: float = 120.0, + ) -> None: + websockets_client, msgpack_numpy = self._require_dependencies() + self._msgpack_numpy = msgpack_numpy + self._packer = msgpack_numpy.Packer() + self._conn = websockets_client.connect( + uri, + compression=None, + max_size=None, + ping_interval=ping_interval, + ping_timeout=ping_timeout, + open_timeout=open_timeout, + close_timeout=close_timeout, + ) + server_metadata = msgpack_numpy.unpackb(self._conn.recv()) + if not isinstance(server_metadata, dict): + raise TypeError(f"Expected dict metadata from {uri}, got {type(server_metadata)!r}") + self._server_metadata = server_metadata + + def get_server_metadata(self) -> dict[str, Any]: + return dict(self._server_metadata) + + def infer(self, obs: dict[str, Any]) -> dict[str, np.ndarray]: + response = self._send_operation("infer", obs) + if not isinstance(response, dict): + raise TypeError(f"Expected dict infer response, got {type(response)!r}") + return {str(key): np.asarray(value, dtype=np.float32) for key, value in response.items()} + + def reset(self, reset_info: dict[str, Any] | None = None) -> str: + response = self._send_operation("reset", dict(reset_info or {})) + return str(response["status"]) + + def close(self) -> None: + self._conn.close() + + def _send_operation(self, endpoint: str, payload: dict[str, Any]) -> Any: + body = dict(payload) + body["endpoint"] = endpoint + self._conn.send(self._packer.pack(body)) + raw = self._conn.recv() + if isinstance(raw, str): + raise RuntimeError(f"OpenPI {endpoint!r} failed: {raw}") + decoded = self._msgpack_numpy.unpackb(raw) + if isinstance(decoded, dict) and decoded.get("type") == "error": + raise RuntimeError(f"OpenPI {endpoint!r} failed: {decoded.get('message')}") + if endpoint == "reset": + if not isinstance(decoded, dict) or decoded.get("status") != "reset successful": + raise RuntimeError(f"Unexpected OpenPI reset response: {decoded!r}") + return decoded + return decoded + + def _merge_http_expectation_kwargs( base: dict[str, Any] | None, *, @@ -1593,6 +1777,92 @@ class OpenAIClientHandler: ) return self._send_websocket_first_json_request("/v1/realtime", cfg) + def send_robot_openpi_ws_request( + self, + request_config: dict[str, Any] | None = None, + ) -> list[OpenPIWebSocketResponse]: + """WebSocket ``/v1/realtime/robot/openpi`` — msgpack metadata plus optional infer/reset ops. + + ``request_config`` keys: + + - ``operations``: optional sequence of ``{"endpoint": "infer"|"reset", "payload": {...}}``. + - ``run_default_policy_session``: when true and ``operations`` is omitted, run infer then reset + using :func:`build_openpi_droid_observation`. + - ``run_dreamzero_policy_session``: when true and ``operations`` is omitted, run the DreamZero + infer/reset/infer sequence using :func:`build_dreamzero_demo_observations`. + - ``video_dir``: required for ``run_dreamzero_policy_session``. + - ``prompt``: language instruction for DreamZero observations (optional). + - ``session_id``: used by the default policy session observation builder. + - ``num_chunks``: DreamZero chunk count (default ``2``). + - ``timeout``: seconds to wait for each inbound msgpack frame (default ``120``). + - ``ping_interval`` / ``ping_timeout``: forwarded to :func:`websockets.sync.client.connect`. + """ + cfg = dict(request_config or {}) + operations_cfg = cfg.get("operations") + if operations_cfg is not None: + operations = [dict(op) for op in operations_cfg] + elif cfg.get("run_dreamzero_policy_session"): + video_dir = cfg.get("video_dir") + if video_dir is None: + raise ValueError("run_dreamzero_policy_session requires video_dir") + session_id = str(cfg.get("session_id", "dreamzero-smoke")) + prompt = str(cfg.get("prompt", DREAMZERO_DEFAULT_PROMPT)) + num_chunks = int(cfg.get("num_chunks", 2)) + observations = build_dreamzero_demo_observations( + load_dreamzero_camera_frames(Path(video_dir)), + prompt=prompt, + session_id=session_id, + num_chunks=num_chunks, + ) + operations = [{"endpoint": "infer", "payload": obs} for obs in observations] + operations.append({"endpoint": "reset", "payload": {}}) + operations.append({"endpoint": "infer", "payload": observations[0]}) + elif cfg.get("run_default_policy_session"): + session_id = str(cfg.get("session_id", "gr00t-smoke")) + operations = [ + {"endpoint": "infer", "payload": build_openpi_droid_observation(session_id=session_id)}, + {"endpoint": "reset", "payload": {}}, + ] + else: + operations = [] + timeout = float(cfg.get("timeout", 120.0)) + ping_interval = float(cfg.get("ping_interval", OpenPIWebSocketSession.DEFAULT_PING_INTERVAL_SECS)) + ping_timeout = float(cfg.get("ping_timeout", OpenPIWebSocketSession.DEFAULT_PING_TIMEOUT_SECS)) + + uri = self._build_ws_url("/v1/realtime/robot/openpi") + session = OpenPIWebSocketSession( + uri, + ping_interval=ping_interval, + ping_timeout=ping_timeout, + open_timeout=timeout, + close_timeout=timeout, + ) + try: + operation_responses: list[Any] = [] + actions: dict[str, np.ndarray] | None = None + action_tensors: list[np.ndarray] = [] + for operation in operations: + endpoint = str(operation["endpoint"]) + payload = dict(operation.get("payload") or {}) + response = session._send_operation(endpoint, payload) + operation_responses.append(response) + if endpoint == "infer": + if isinstance(response, dict): + actions = {str(key): np.asarray(value, dtype=np.float32) for key, value in response.items()} + else: + action_tensors.append(np.asarray(response, dtype=np.float32)) + finally: + session.close() + + return [ + OpenPIWebSocketResponse( + server_metadata=session.get_server_metadata(), + operation_responses=operation_responses, + actions=actions, + action_tensors=action_tensors or None, + ) + ] + def send_omni_request(self, request_config: dict[str, Any], request_num: int = 1) -> list[OmniResponse]: """Chat completions via the OpenAI Python SDK (not raw HTTP).""" responses: list[OmniResponse] = [] @@ -2946,6 +3216,14 @@ __all__ = [ "DiffusionResponse", "HttpResponse", "WebSocketJsonResponse", + "OpenPIWebSocketResponse", + "build_openpi_droid_observation", + "build_dreamzero_demo_observations", + "DREAMZERO_ACTION_DIM", + "DREAMZERO_ACTION_HORIZON", + "DREAMZERO_CAMERA_FILES", + "load_dreamzero_camera_frames", + "OpenPIWebSocketSession", "OmniResponse", "OmniRunner", "OmniRunnerHandler", diff --git a/tests/helpers/test_assertions.py b/tests/helpers/tests/test_assertions.py similarity index 92% rename from tests/helpers/test_assertions.py rename to tests/helpers/tests/test_assertions.py index 0bfa7f593..8293349fa 100644 --- a/tests/helpers/test_assertions.py +++ b/tests/helpers/tests/test_assertions.py @@ -5,6 +5,8 @@ import pytest from tests.helpers.assertions import _assert_transcript_matches +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + def test_short_transcript_repeat_passes_containment_fallback(): _assert_transcript_matches( diff --git a/tests/test_pydub_removal.py b/tests/helpers/tests/test_media_pydub_removal.py similarity index 85% rename from tests/test_pydub_removal.py rename to tests/helpers/tests/test_media_pydub_removal.py index ae65bdab2..60e5cab65 100644 --- a/tests/test_pydub_removal.py +++ b/tests/helpers/tests/test_media_pydub_removal.py @@ -7,13 +7,19 @@ identical results to the old pydub-based code for all three paths: 3. TTS WER PCM export: resample to 24000Hz mono s16 """ +import base64 import io import wave import av import numpy as np +import pytest import soundfile as sf +from tests.helpers.media import _merge_base64_audio_to_segment + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + def _make_wav_bytes(sample_rate=16000, channels=1, sample_width=2, duration_sec=1.0): """Generate a WAV file with a sine wave, return (wav_bytes, pcm_bytes, params).""" @@ -69,8 +75,6 @@ def test_wav_path_duration_and_frames(): f"sr={sr} ch={ch} dur={dur}: frames {audio_frames} != {expected_frames}" ) - print("[PASS] test_wav_path_duration_and_frames") - def test_non_wav_path_duration_av(): """Path 2: non-WAV audio → duration & frames via av.""" @@ -86,8 +90,6 @@ def test_non_wav_path_duration_av(): # OGG encoding may add padding, allow small tolerance assert abs(audio_duration_sec - dur) < 0.05, f"dur={dur}: got {audio_duration_sec}" - print("[PASS] test_non_wav_path_duration_av") - def test_tts_pcm_export_wav_path(): """Path 3a: WAV PCM → resample to 24000Hz mono s16.""" @@ -114,8 +116,6 @@ def test_tts_pcm_export_wav_path(): assert abs(actual_samples - expected_samples) < 10, f"expected ~{expected_samples} samples, got {actual_samples}" assert len(pcm_array.shape) == 1, "should be mono (1D array)" - print("[PASS] test_tts_pcm_export_wav_path") - def test_tts_pcm_export_non_wav_path(): """Path 3b: non-WAV → resample to 24000Hz mono s16 via av.AudioResampler.""" @@ -135,22 +135,13 @@ def test_tts_pcm_export_non_wav_path(): actual_samples = len(result_bytes) // 2 assert abs(actual_samples - expected_samples) < 500, f"expected ~{expected_samples} samples, got {actual_samples}" - print("[PASS] test_tts_pcm_export_non_wav_path") - def test_media_merge_and_export(): """media.py: _merge_base64_audio_to_segment decode + concat + export WAV.""" - import base64 as b64mod - import sys - - sys.path.insert(0, ".") - from tests.helpers.media import _merge_base64_audio_to_segment - - # Create two WAV chunks as base64 chunks_b64 = [] for dur in [0.5, 0.3]: wav_bytes, _, _ = _make_wav_bytes(sample_rate=16000, channels=1, sample_width=2, duration_sec=dur) - chunks_b64.append(b64mod.b64encode(wav_bytes).decode()) + chunks_b64.append(base64.b64encode(wav_bytes).decode()) merged = _merge_base64_audio_to_segment(chunks_b64) @@ -164,19 +155,3 @@ def test_media_merge_and_export(): out_data, out_sr = sf.read(io.BytesIO(wav_out)) actual_dur = len(out_data) / out_sr assert abs(actual_dur - 0.8) < 0.05, f"expected ~0.8s merged duration, got {actual_dur}" - - print(f"[PASS] test_media_merge_and_export (duration={actual_dur:.3f}s, wav_size={len(wav_out)} bytes)") - - -if __name__ == "__main__": - test_wav_path_duration_and_frames() - test_non_wav_path_duration_av() - import importlib.util - - if importlib.util.find_spec("torch") and importlib.util.find_spec("torchaudio"): - test_tts_pcm_export_wav_path() - else: - print("[SKIP] test_tts_pcm_export_wav_path (torch/torchaudio not available)") - test_tts_pcm_export_non_wav_path() - test_media_merge_and_export() - print("\nAll tests passed.") diff --git a/tests/helpers/test_runtime_stage_cli.py b/tests/helpers/tests/test_runtime_stage_cli.py similarity index 95% rename from tests/helpers/test_runtime_stage_cli.py rename to tests/helpers/tests/test_runtime_stage_cli.py index d9b50c85f..b127eac13 100644 --- a/tests/helpers/test_runtime_stage_cli.py +++ b/tests/helpers/tests/test_runtime_stage_cli.py @@ -4,8 +4,12 @@ from __future__ import annotations from pathlib import Path +import pytest + from tests.helpers.runtime import OmniServerStageCli +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + def _write_stage_config(tmp_path: Path) -> str: path = tmp_path / "stages.yaml" diff --git a/tests/model_executor/models/cosyvoice3/test_cosyvoice3_components.py b/tests/model_executor/models/cosyvoice3/test_cosyvoice3_components.py index bf2261cb9..2a4cb2f1e 100644 --- a/tests/model_executor/models/cosyvoice3/test_cosyvoice3_components.py +++ b/tests/model_executor/models/cosyvoice3/test_cosyvoice3_components.py @@ -10,6 +10,8 @@ import torch.nn as nn from tests.helpers.mark import hardware_test +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + class TestPreLookaheadLayer: """Tests for PreLookaheadLayer.""" diff --git a/tests/model_executor/models/dynin_omni/test_dynin_omni_token2audio.py b/tests/model_executor/models/dynin_omni/test_dynin_omni_token2audio.py index d77b28cd5..d2138541f 100644 --- a/tests/model_executor/models/dynin_omni/test_dynin_omni_token2audio.py +++ b/tests/model_executor/models/dynin_omni/test_dynin_omni_token2audio.py @@ -1,16 +1,34 @@ from __future__ import annotations +import pytest from transformers import PreTrainedModel from vllm_omni.model_executor.models.dynin_omni.dynin_omni_token2audio import ( _ensure_transformers_tied_weights_compat, ) +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + def test_transformers_tied_weights_compat(monkeypatch): + """Compat shim must not permanently mutate ``PreTrainedModel``. + + ``monkeypatch.delattr(..., raising=False)`` is a no-op undo when the attr + was absent. ``_ensure_transformers_tied_weights_compat`` then assigns a + read-only ``property`` on the class that later PreTrainedModel subclasses + (e.g. qwen3 TTS mask-cache) can trip over with + ``AttributeError: property ... has no setter``. + """ monkeypatch.delattr(PreTrainedModel, "all_tied_weights_keys", raising=False) _ensure_transformers_tied_weights_compat() + installed = PreTrainedModel.__dict__.get("all_tied_weights_keys") + assert installed is not None + + # Re-own the installed descriptor through monkeypatch so teardown always + # removes/restores it, even when the attr did not exist before this test. + delattr(PreTrainedModel, "all_tied_weights_keys") + monkeypatch.setattr(PreTrainedModel, "all_tied_weights_keys", installed, raising=False) model = PreTrainedModel.__new__(PreTrainedModel) model._tied_weights_keys = ["decoder.weight"] diff --git a/tests/model_executor/models/fish_speech/test_fish_speech_fast_ar_runtime_buffers.py b/tests/model_executor/models/fish_speech/test_fish_speech_fast_ar_runtime_buffers.py index 41a29ca63..9586ad053 100644 --- a/tests/model_executor/models/fish_speech/test_fish_speech_fast_ar_runtime_buffers.py +++ b/tests/model_executor/models/fish_speech/test_fish_speech_fast_ar_runtime_buffers.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import pytest import torch import torch.nn as nn @@ -13,6 +14,8 @@ from vllm_omni.model_executor.models.fish_speech.fish_speech_slow_ar import ( FishSpeechSlowARForConditionalGeneration, ) +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + def test_fast_ar_reuses_dense_position_id_buffer(monkeypatch): fast_config = FishSpeechFastARConfig( diff --git a/tests/unit/higgs_audio_v3/test_higgs_audio_v3.py b/tests/model_executor/models/higgs_audio_v3/test_higgs_audio_v3.py similarity index 97% rename from tests/unit/higgs_audio_v3/test_higgs_audio_v3.py rename to tests/model_executor/models/higgs_audio_v3/test_higgs_audio_v3.py index 1d4bdb48d..6d81f300a 100755 --- a/tests/unit/higgs_audio_v3/test_higgs_audio_v3.py +++ b/tests/model_executor/models/higgs_audio_v3/test_higgs_audio_v3.py @@ -8,6 +8,7 @@ without requiring the actual checkpoint or GPU. import asyncio import json +import os import time from collections import OrderedDict, defaultdict from types import SimpleNamespace @@ -15,6 +16,11 @@ from types import SimpleNamespace import pytest import torch +from tests.helpers.stage_config import get_deploy_config_path + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + # ---- AC-1: Configuration ---- @@ -240,6 +246,37 @@ class TestDelayPatternBehavior: class TestSamplerMethods: """Test the actual sampler and batched delay masking methods.""" + @pytest.fixture(autouse=True) + def _stub_apply_top_k_top_p_on_cpu(self, monkeypatch): + # File is marked cpu; vLLM's apply_top_k_top_p uses Triton and rejects + # CPU tensors. Use a pure-PyTorch fallback that still honors top-k/top-p. + def _cpu_apply_top_k_top_p(logits, top_ks, top_ps): + out = logits.clone() + vocab = out.shape[-1] + top_ks = top_ks.reshape(-1) + top_ps = top_ps.reshape(-1) + for row in range(out.shape[0]): + k = int(top_ks[row].item()) + p = float(top_ps[row].item()) + if 0 < k < vocab: + threshold = torch.topk(out[row], k).values[-1] + out[row].masked_fill_(out[row] < threshold, float("-inf")) + if 0.0 < p < 1.0: + sorted_logits, sorted_idx = torch.sort(out[row], descending=True) + probs = torch.softmax(sorted_logits, dim=-1) + cumprobs = torch.cumsum(probs, dim=-1) + # Keep the first token that exceeds p; drop the rest. + to_remove = cumprobs > p + to_remove[1:] = to_remove[:-1].clone() + to_remove[0] = False + out[row, sorted_idx[to_remove]] = float("-inf") + return out + + monkeypatch.setattr( + "vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker.apply_top_k_top_p", + _cpu_apply_top_k_top_p, + ) + def _make_minimal_talker(self): """Create a minimal talker-like object with sampler/masking methods.""" from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod @@ -1290,17 +1327,12 @@ class TestRegistry: assert "higgs_multimodal_qwen3" in OMNI_PIPELINES def test_deploy_yaml_exists(self): - import os - - # __file__ = tests/unit/higgs_audio_v3/test_*.py → 4 dirnames to repo root - repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) - deploy_dir = os.path.join(repo_root, "vllm_omni", "deploy") for name in ( "higgs_multimodal_qwen3.yaml", "higgs_multimodal_qwen3_high_throughput.yaml", "higgs_multimodal_qwen3_low_latency.yaml", ): - yaml_path = os.path.join(deploy_dir, name) + yaml_path = get_deploy_config_path(name) assert os.path.isfile(yaml_path), f"Deploy YAML not found at {yaml_path}" diff --git a/tests/unit/mammoth_moda2/test_mammoth_moda2_config.py b/tests/model_executor/models/mammoth_moda2/test_mammoth_moda2_config.py similarity index 98% rename from tests/unit/mammoth_moda2/test_mammoth_moda2_config.py rename to tests/model_executor/models/mammoth_moda2/test_mammoth_moda2_config.py index 6e35a94f6..05d5dbdde 100644 --- a/tests/unit/mammoth_moda2/test_mammoth_moda2_config.py +++ b/tests/model_executor/models/mammoth_moda2/test_mammoth_moda2_config.py @@ -23,7 +23,6 @@ from vllm_omni.transformers_utils.configs.mammoth_moda2 import ( pytestmark = [pytest.mark.core_model, pytest.mark.cpu] -@pytest.mark.cpu class TestMammothmoda2Config: """Tests for Mammothmoda2Config class.""" @@ -75,7 +74,6 @@ class TestMammothmoda2Config: assert config.llm_config is not None -@pytest.mark.cpu class TestMammothmoda2Qwen2_5_VLConfig: """Tests for Mammothmoda2Qwen2_5_VLConfig class.""" @@ -95,7 +93,6 @@ class TestMammothmoda2Qwen2_5_VLConfig: assert hasattr(text_config, "video_token_id") -@pytest.mark.cpu class TestMammothmoda2Qwen3VLConfig: def test_dev_config_uses_qwen3_vl_subconfigs(self): config = Mammothmoda2Config( diff --git a/tests/model_executor/models/moss_tts/test_codec_cudagraph.py b/tests/model_executor/models/moss_tts/test_codec_cudagraph.py index 49d6fa411..91a853d57 100644 --- a/tests/model_executor/models/moss_tts/test_codec_cudagraph.py +++ b/tests/model_executor/models/moss_tts/test_codec_cudagraph.py @@ -1,14 +1,15 @@ """Tests for MossTTSCUDAGraphCodecWrapper numerical equivalence. Verifies that CUDA Graph-accelerated decoding produces results equivalent -to eager mode, with special attention to the two-argument _decode interface -(codes [NQ, 1, T] + lengths [1]) and the NQ-first input convention. +to eager mode, with special attention to the two-argument _decode +interface (codes [NQ, 1, T] + lengths [1]) and the NQ-first input convention. """ import pytest import torch import torch.nn as nn +from tests.helpers.mark import hardware_marks from vllm_omni.model_executor.models.moss_tts.audio_tokenizer import ( MossAudioTokenizerDecoderOutput, ) @@ -16,8 +17,10 @@ from vllm_omni.model_executor.models.moss_tts.moss_codec_cudagraph import ( MossTTSCUDAGraphCodecWrapper, ) -pytestmark = [pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")] - +pytestmark = [ + pytest.mark.core_model, + *hardware_marks(res={"cuda": "L4"}, num_cards=1), +] DEVICE = torch.device("cuda:0") NUM_QUANTIZERS = 8 DOWNSAMPLE_RATE = 4 # synthetic; real checkpoint uses 1920 @@ -31,8 +34,9 @@ DOWNSAMPLE_RATE = 4 # synthetic; real checkpoint uses 1920 class SyntheticCodecModel(nn.Module): """Minimal stand-in for MossAudioTokenizerModel. - Exposes the same two-argument _decode(codes, lengths) interface and - returns a MossAudioTokenizerDecoderOutput, mirroring the real model. + Exposes the same two-argument ``_decode(codes, lengths)`` interface + used by ``MossTTSCUDAGraphCodecWrapper`` and returns a + MossAudioTokenizerDecoderOutput. Input: codes: [NQ, B, T] long — RVQ codes diff --git a/tests/model_executor/models/moss_tts/test_per_request_codes.py b/tests/model_executor/models/moss_tts/test_per_request_codes.py index b35346c2f..91804ee15 100644 --- a/tests/model_executor/models/moss_tts/test_per_request_codes.py +++ b/tests/model_executor/models/moss_tts/test_per_request_codes.py @@ -23,6 +23,9 @@ import functools import pytest +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + torch = pytest.importorskip("torch") N_VQ = 4 diff --git a/tests/model_executor/models/qwen3_tts/test_cuda_graph_decoder.py b/tests/model_executor/models/qwen3_tts/test_cuda_graph_decoder.py index 0f77455d7..f3e8d0166 100644 --- a/tests/model_executor/models/qwen3_tts/test_cuda_graph_decoder.py +++ b/tests/model_executor/models/qwen3_tts/test_cuda_graph_decoder.py @@ -20,8 +20,9 @@ import pytest import torch import torch.nn as nn -pytestmark = [pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")] +from tests.helpers.mark import hardware_test +pytestmark = [pytest.mark.core_model] DEVICE = torch.device("cuda:0") NUM_QUANTIZERS = 8 TOTAL_UPSAMPLE = 4 @@ -112,6 +113,7 @@ def _random_codes(seq_len, batch_size=1, device=DEVICE): # ────────────────────────────────────────────────────────────────── +@hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("seq_len", [25, 50, 100]) def test_exact_size_numerical_equivalence(decoder, wrapper, seq_len): """When input exactly matches a capture size, output must be bit-identical.""" @@ -127,6 +129,7 @@ def test_exact_size_numerical_equivalence(decoder, wrapper, seq_len): # ────────────────────────────────────────────────────────────────── +@hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("seq_len", [10, 30, 47, 73, 99]) def test_padded_output_shape_and_length(decoder, wrapper, seq_len): """Padded decode must return output trimmed to actual input length.""" @@ -139,6 +142,7 @@ def test_padded_output_shape_and_length(decoder, wrapper, seq_len): assert graph_out.shape[-1] == expected_len +@hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("seq_len", [10, 30, 47, 73, 99]) def test_padded_interior_positions_close(decoder, wrapper, seq_len): """Interior positions (away from padding boundary) should be very close. @@ -160,6 +164,7 @@ def test_padded_interior_positions_close(decoder, wrapper, seq_len): torch.testing.assert_close(interior_graph, interior_eager, atol=1e-5, rtol=1e-5) +@hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("seq_len", [10, 30, 47, 73, 99]) def test_padded_output_bounded(decoder, wrapper, seq_len): """Padded output values must remain in [-1, 1] and max diff should be bounded.""" @@ -175,6 +180,7 @@ def test_padded_output_bounded(decoder, wrapper, seq_len): assert max_diff < 1.0, f"Max diff {max_diff} exceeds bound" +@hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("seq_len", [10, 30, 47, 73, 99]) def test_padded_short_output_decode_length_matches_eager(seq_len): """Streaming ``decode()`` of a shorter-than-nominal decoder must match eager length. @@ -210,6 +216,7 @@ def test_padded_short_output_decode_length_matches_eager(seq_len): torch.testing.assert_close(graph_out[..., :-boundary], eager_out[..., :-boundary], atol=1e-5, rtol=1e-5) +@hardware_test(res={"cuda": "L4"}, num_cards=1) def test_compiled_padded_short_output_length_matches_eager(monkeypatch): """Compiled replay path must also trim short-output decoders to eager length. @@ -251,6 +258,7 @@ def test_compiled_padded_short_output_length_matches_eager(monkeypatch): # ────────────────────────────────────────────────────────────────── +@hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("seq_len", [101, 150, 200]) def test_fallback_eager_exact_match(decoder, wrapper, seq_len): """Input larger than all capture sizes falls back to eager -> bit-identical.""" @@ -266,6 +274,7 @@ def test_fallback_eager_exact_match(decoder, wrapper, seq_len): # ────────────────────────────────────────────────────────────────── +@hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("total_len", [60, 100, 150, 250]) def test_chunked_decode_shape_match(decoder, wrapper, total_len): """Chunked decode output shape must match between eager and graph modes.""" @@ -279,6 +288,7 @@ def test_chunked_decode_shape_match(decoder, wrapper, total_len): assert eager_out.shape == graph_out.shape +@hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("total_len", [50, 100]) def test_chunked_decode_exact_size_equivalence(decoder, wrapper, total_len): """Chunked decode with chunks matching capture sizes should be bit-identical.""" @@ -293,6 +303,7 @@ def test_chunked_decode_exact_size_equivalence(decoder, wrapper, total_len): torch.testing.assert_close(graph_out, eager_out, atol=0, rtol=0) +@hardware_test(res={"cuda": "L4"}, num_cards=1) def test_chunked_decode_output_survives_later_replay(wrapper): """Chunked output must not alias graph static buffers overwritten by later replays.""" codes = _random_codes(100) @@ -307,6 +318,7 @@ def test_chunked_decode_output_survives_later_replay(wrapper): torch.testing.assert_close(graph_out, expected, atol=0, rtol=0) +@hardware_test(res={"cuda": "L4"}, num_cards=1) def test_chunked_decode_preserves_short_chunk_concat_semantics(): """Chunked decode must allow shorter-than-nominal chunk outputs. @@ -336,6 +348,7 @@ def test_chunked_decode_preserves_short_chunk_concat_semantics(): torch.testing.assert_close(graph_out, eager_out, atol=0, rtol=0) +@hardware_test(res={"cuda": "L4"}, num_cards=1) def test_batched_chunked_decode_variable_lengths_matches_per_request_eager(decoder, wrapper): """Variable-length chunk batching should match independent chunked decodes.""" long_codes = _random_codes(100) @@ -380,6 +393,7 @@ def _eager_chunked(decoder, codes, chunk_size, left_context_size): # ────────────────────────────────────────────────────────────────── +@hardware_test(res={"cuda": "L4"}, num_cards=1) def test_single_frame(decoder, wrapper): """Single-frame input (seq_len=1) should work with padding.""" codes = _random_codes(1) @@ -390,6 +404,7 @@ def test_single_frame(decoder, wrapper): assert graph_out.shape[-1] == TOTAL_UPSAMPLE +@hardware_test(res={"cuda": "L4"}, num_cards=1) def test_disabled_wrapper_matches_eager(decoder, wrapper): """Disabled wrapper should produce bit-identical output to eager.""" codes = _random_codes(30) @@ -401,6 +416,7 @@ def test_disabled_wrapper_matches_eager(decoder, wrapper): torch.testing.assert_close(graph_out, eager_out, atol=0, rtol=0) +@hardware_test(res={"cuda": "L4"}, num_cards=1) def test_batch_size_gt1_uses_matching_graph(decoder, wrapper): """Captured batch size > 1 should replay a matching graph.""" assert (2, 25) in wrapper.graphs @@ -411,6 +427,7 @@ def test_batch_size_gt1_uses_matching_graph(decoder, wrapper): torch.testing.assert_close(graph_out, eager_out, atol=0, rtol=0) +@hardware_test(res={"cuda": "L4"}, num_cards=1) def test_uncaptured_batch_size_falls_back(decoder, wrapper): """Uncaptured batch sizes should fall back to eager.""" assert (3, 25) not in wrapper.graphs @@ -421,6 +438,7 @@ def test_uncaptured_batch_size_falls_back(decoder, wrapper): torch.testing.assert_close(graph_out, eager_out, atol=0, rtol=0) +@hardware_test(res={"cuda": "L4"}, num_cards=1) def test_extra_capture_shape_uses_sparse_graph(decoder): """Extra capture shapes should not expand to a full batch x size product.""" sparse_wrapper = CUDAGraphDecoderWrapper( @@ -444,6 +462,7 @@ def test_extra_capture_shape_uses_sparse_graph(decoder): torch.testing.assert_close(graph_out, eager_out, atol=0, rtol=0) +@hardware_test(res={"cuda": "L4"}, num_cards=1) def test_compile_shape_supports_exact_and_padded_buckets(decoder, monkeypatch): """Configured torch.compile shapes should replay exact and padded CUDA Graph buckets.""" @@ -490,6 +509,7 @@ def test_compile_shape_supports_exact_and_padded_buckets(decoder, monkeypatch): assert compile_kwargs["dynamic"] is False +@hardware_test(res={"cuda": "L4"}, num_cards=1) def test_deterministic_across_calls(decoder, wrapper): """Same input should produce identical CUDA graph output across calls.""" codes = _random_codes(30) @@ -504,6 +524,7 @@ def test_deterministic_across_calls(decoder, wrapper): # ────────────────────────────────────────────────────────────────── +@pytest.mark.cpu @pytest.mark.parametrize( "kwargs,expected_in,not_expected", [ @@ -545,6 +566,7 @@ def test_compute_capture_sizes(kwargs, expected_in, not_expected): # ────────────────────────────────────────────────────────────────── +@hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize( "batch,channels,seq_len", [(2, 64, 1000), (1, 32, 1), (1, 32, 7), (1, 32, 128), (1, 32, 1024), (1, 32, 4096)], diff --git a/tests/model_executor/models/qwen3_tts/test_qwen3_tts_talker_preprocess.py b/tests/model_executor/models/qwen3_tts/test_qwen3_tts_talker_preprocess.py index 7358c7893..c76583a4e 100644 --- a/tests/model_executor/models/qwen3_tts/test_qwen3_tts_talker_preprocess.py +++ b/tests/model_executor/models/qwen3_tts/test_qwen3_tts_talker_preprocess.py @@ -17,6 +17,8 @@ from vllm_omni.model_executor.models.qwen3_tts.qwen3_tts_talker import ( Qwen3TTSTalkerForConditionalGeneration, ) +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + def _make_minimal_talker( tts_pad_embed: torch.Tensor | None = None, diff --git a/tests/test_tts_custom_voice_profiles.py b/tests/model_executor/models/qwen3_tts/test_tts_custom_voice_profiles.py similarity index 100% rename from tests/test_tts_custom_voice_profiles.py rename to tests/model_executor/models/qwen3_tts/test_tts_custom_voice_profiles.py diff --git a/tests/model_executor/models/voxcpm2/test_runner_assisted_unified_graph_static.py b/tests/model_executor/models/voxcpm2/test_runner_assisted_unified_graph_static.py index 8154ddd65..0007f021a 100644 --- a/tests/model_executor/models/voxcpm2/test_runner_assisted_unified_graph_static.py +++ b/tests/model_executor/models/voxcpm2/test_runner_assisted_unified_graph_static.py @@ -8,6 +8,11 @@ lightweight local checkout. Runtime audio quality still requires a CUDA test. from pathlib import Path +import pytest + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + REPO_ROOT = Path(__file__).resolve().parents[4] RUNNER = REPO_ROOT / "vllm_omni" / "worker" / "gpu_ar_model_runner.py" RUNNER_ASSISTED_METADATA = REPO_ROOT / "vllm_omni" / "worker" / "runner_assisted_metadata.py" diff --git a/tests/model_executor/models/voxcpm2/test_talker_state_eviction.py b/tests/model_executor/models/voxcpm2/test_talker_state_eviction.py index f5a94376a..3938051d3 100644 --- a/tests/model_executor/models/voxcpm2/test_talker_state_eviction.py +++ b/tests/model_executor/models/voxcpm2/test_talker_state_eviction.py @@ -9,6 +9,9 @@ from types import SimpleNamespace import pytest +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + torch = pytest.importorskip("torch") diff --git a/tests/model_executor/stage_input_processors/test_cosyvoice3_stage_input_processors.py b/tests/model_executor/stage_input_processors/test_cosyvoice3_stage_input_processors.py index 5309c7675..d5971efe7 100644 --- a/tests/model_executor/stage_input_processors/test_cosyvoice3_stage_input_processors.py +++ b/tests/model_executor/stage_input_processors/test_cosyvoice3_stage_input_processors.py @@ -4,6 +4,7 @@ from collections import defaultdict from types import SimpleNamespace +import pytest import torch from vllm_omni.model_executor.stage_input_processors.cosyvoice3 import ( @@ -12,6 +13,8 @@ from vllm_omni.model_executor.stage_input_processors.cosyvoice3 import ( text2flow_token_only, ) +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + def _source_output(request_id: str, prompt_ids: list[int], out_ids: list[int], mm: dict, finished: bool = True): return SimpleNamespace( diff --git a/tests/model_executor/stage_input_processors/test_indextts2.py b/tests/model_executor/stage_input_processors/test_indextts2.py index 11468603e..1380b99ff 100644 --- a/tests/model_executor/stage_input_processors/test_indextts2.py +++ b/tests/model_executor/stage_input_processors/test_indextts2.py @@ -3,6 +3,7 @@ from types import SimpleNamespace +import pytest import torch from vllm_omni.model_executor.stage_input_processors.indextts2 import ( @@ -11,6 +12,9 @@ from vllm_omni.model_executor.stage_input_processors.indextts2 import ( talker2s2mel_token_only, ) +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + STOP_MEL_TOKEN = 8193 LATENT_DIM = 16 diff --git a/tests/model_executor/stage_input_processors/test_qwen2_5_omni_thinker2talker.py b/tests/model_executor/stage_input_processors/test_qwen2_5_omni_thinker2talker.py index 0cb61a972..3ecae3d56 100644 --- a/tests/model_executor/stage_input_processors/test_qwen2_5_omni_thinker2talker.py +++ b/tests/model_executor/stage_input_processors/test_qwen2_5_omni_thinker2talker.py @@ -20,6 +20,8 @@ from vllm_omni.model_executor.stage_input_processors.qwen2_5_omni import ( thinker2talker_full_payload, ) +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + def _make_request( prompt_token_ids, diff --git a/tests/plugin/__init__.py b/tests/plugin/__init__.py deleted file mode 100644 index 988131360..000000000 --- a/tests/plugin/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 diff --git a/tests/profile/test_omni_torch_profiler.py b/tests/profile/test_omni_torch_profiler.py index 3920078af..7bfc8f8cc 100644 --- a/tests/profile/test_omni_torch_profiler.py +++ b/tests/profile/test_omni_torch_profiler.py @@ -12,6 +12,8 @@ from openpyxl import load_workbook import vllm_omni.profiler.omni_torch_profiler as profiler_mod from vllm_omni.profiler.omni_torch_profiler import OmniTorchProfilerWrapper +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + @pytest.fixture(autouse=True) def patch_worker_profiler_init(monkeypatch): diff --git a/tests/quantization/test_quack_fp8_gate.py b/tests/quantization/test_quack_fp8_gate.py index 4a8848ffa..843f4ff5f 100644 --- a/tests/quantization/test_quack_fp8_gate.py +++ b/tests/quantization/test_quack_fp8_gate.py @@ -20,6 +20,9 @@ import torch from vllm_omni.quantization import quack_fp8 +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + _ENV = "VLLM_OMNI_USE_QUACK_FP8" diff --git a/tests/reasoning/test_step_audio_reasoning_parser.py b/tests/reasoning/test_step_audio_reasoning_parser.py index 7eb9fa17a..11105265c 100644 --- a/tests/reasoning/test_step_audio_reasoning_parser.py +++ b/tests/reasoning/test_step_audio_reasoning_parser.py @@ -21,6 +21,9 @@ from unittest.mock import MagicMock import pytest +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + # --------------------------------------------------------------------------- # Bootstrap minimal vLLM stubs so the parser can be imported # --------------------------------------------------------------------------- @@ -784,8 +787,8 @@ class TestExtractReasoningStreaming: """When buffered text turns out not to be a marker prefix, it is emitted as reasoning on the next call.""" # "reason<" is buffered because "<" could start a marker. - # Next delta ">" doesn't form a marker, so the buffer is - # flushed as reasoning. + # Next delta ">" doesn't form a marker, so the buffer ("<") is + # prepended to the next delta and flushed as reasoning: "<> more". parser = multi_token_parser deltas_and_ids = [ (TS, [100]), @@ -794,7 +797,7 @@ class TestExtractReasoningStreaming: ("> more", [8]), # ">" doesn't complete any marker ] r, c = _stream_extract(parser, deltas_and_ids) - assert r == "reason> more" + assert r == "reason<> more" assert c == "" def test_multi_token_end_with_reasoning_before_buffer(self, multi_token_parser): @@ -837,7 +840,7 @@ class TestExtractReasoningStreaming: ("> not a marker", [8]), ] r, c = _stream_extract(parser, deltas_and_ids) - assert r == "reason> not a marker" + assert r == "reason<> not a marker" assert c == "" diff --git a/tests/test_generate_nightly_perf_excel.py b/tests/tools/test_generate_nightly_perf_excel.py similarity index 96% rename from tests/test_generate_nightly_perf_excel.py rename to tests/tools/test_generate_nightly_perf_excel.py index 6658b89b8..80b27fbf5 100644 --- a/tests/test_generate_nightly_perf_excel.py +++ b/tests/tools/test_generate_nightly_perf_excel.py @@ -10,7 +10,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu] def _load_excel_module(): - repo_root = Path(__file__).resolve().parents[1] + repo_root = Path(__file__).resolve().parents[2] module_path = repo_root / "tools" / "nightly" / "generate_nightly_perf_excel.py" spec = importlib.util.spec_from_file_location("generate_nightly_perf_excel", module_path) module = importlib.util.module_from_spec(spec) @@ -27,7 +27,7 @@ def _cell_value_by_header(ws, header_name: str, row_idx: int = 2): def test_generate_excel_report_with_perf_templates(tmp_path: Path): module = _load_excel_module() - repo_root = Path(__file__).resolve().parents[1] + repo_root = Path(__file__).resolve().parents[2] perf_scripts_dir = repo_root / "tests" / "dfx" / "perf" / "scripts" omni_template_path = perf_scripts_dir / "result_omni_template.json" diff --git a/tests/test_generate_nightly_perf_html.py b/tests/tools/test_generate_nightly_perf_html.py similarity index 95% rename from tests/test_generate_nightly_perf_html.py rename to tests/tools/test_generate_nightly_perf_html.py index 7a05165da..4592c2183 100644 --- a/tests/test_generate_nightly_perf_html.py +++ b/tests/tools/test_generate_nightly_perf_html.py @@ -9,7 +9,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu] def _load_html_module(): - repo_root = Path(__file__).resolve().parents[1] + repo_root = Path(__file__).resolve().parents[2] module_path = repo_root / "tools" / "nightly" / "generate_nightly_perf_html.py" spec = importlib.util.spec_from_file_location("generate_nightly_perf_html", module_path) module = importlib.util.module_from_spec(spec) @@ -20,7 +20,7 @@ def _load_html_module(): def test_generate_html_report_with_perf_templates(tmp_path: Path): module = _load_html_module() - repo_root = Path(__file__).resolve().parents[1] + repo_root = Path(__file__).resolve().parents[2] perf_scripts_dir = repo_root / "tests" / "dfx" / "perf" / "scripts" omni_template_path = perf_scripts_dir / "result_omni_template.json" diff --git a/tests/unit/higgs_audio_v3/__init__.py b/tests/unit/higgs_audio_v3/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/utils/test_audio.py b/tests/utils/test_audio.py index 0e483e646..ff29b2d04 100644 --- a/tests/utils/test_audio.py +++ b/tests/utils/test_audio.py @@ -9,6 +9,9 @@ import torch from vllm_omni.utils.audio import mel_filter_bank, peak_normalize +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + # Parameter combinations used across the codebase. _PARAM_SETS = [ # Qwen3-TTS talker / speaker encoder (sr=24000) diff --git a/tests/test_speaker_cache.py b/tests/utils/test_speaker_cache.py similarity index 100% rename from tests/test_speaker_cache.py rename to tests/utils/test_speaker_cache.py diff --git a/tests/test_speaker_cache_integration.py b/tests/utils/test_speaker_cache_integration.py similarity index 100% rename from tests/test_speaker_cache_integration.py rename to tests/utils/test_speaker_cache_integration.py diff --git a/tests/utils/test_tracking_parser.py b/tests/utils/test_tracking_parser.py index 00882744c..13c405af7 100644 --- a/tests/utils/test_tracking_parser.py +++ b/tests/utils/test_tracking_parser.py @@ -23,6 +23,9 @@ from vllm_omni.utils.tracking_parser import ( build_shadow_kwargs, ) +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + ### Fake pipeline/deploy config for integration tests _TEST_MODEL = "_test_tracking" diff --git a/tests/test_version.py b/tests/utils/test_version.py similarity index 98% rename from tests/test_version.py rename to tests/utils/test_version.py index 911a79611..0c666133c 100644 --- a/tests/test_version.py +++ b/tests/utils/test_version.py @@ -9,6 +9,8 @@ import pytest from vllm_omni.version import warn_if_misaligned_vllm_version +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + @mock.patch("vllm_omni.version.__version_tuple__", (0, 19, 0)) @mock.patch("vllm_omni.version.__version__", "0.19.0") diff --git a/tools/nightly/buildkite_testcase_statistics.py b/tools/nightly/buildkite_testcase_statistics.py index 7a4f9bff3..740f896f5 100644 --- a/tools/nightly/buildkite_testcase_statistics.py +++ b/tools/nightly/buildkite_testcase_statistics.py @@ -903,11 +903,11 @@ def _sample_preview_stats() -> list[tuple]: "test-ready", None, "Custom Pipeline Test", - ["tests/e2e/offline_inference/custom_pipeline/test_cp.py"], + ["tests/e2e/features/custom_pipeline/test_cp.py"], False, [], 1, - ["tests/e2e/offline_inference/custom_pipeline/test_cp.py::test_cp_one"], + ["tests/e2e/features/custom_pipeline/test_cp.py::test_cp_one"], [""], set(), ), diff --git a/tests/e2e/offline_inference/stage_configs/step_audio2_ci.yaml b/vllm_omni/deploy/step_audio2_thinker_ci.yaml similarity index 93% rename from tests/e2e/offline_inference/stage_configs/step_audio2_ci.yaml rename to vllm_omni/deploy/step_audio2_thinker_ci.yaml index f9dd4f9af..7856ebda9 100644 --- a/tests/e2e/offline_inference/stage_configs/step_audio2_ci.yaml +++ b/vllm_omni/deploy/step_audio2_thinker_ci.yaml @@ -1,4 +1,4 @@ -# Stage config for Step-Audio2 CI tests +# Stage config for Step-Audio2 CI tests (offline inference, thinker-only). # Single Thinker stage (audio/text understanding → text). ASR/text-output # tests don't need Token2Wav; a two-stage config would deadlock on stage-1 skip. diff --git a/vllm_omni/engine/arg_utils.py b/vllm_omni/engine/arg_utils.py index 924f41c3f..ed3f5464d 100644 --- a/vllm_omni/engine/arg_utils.py +++ b/vllm_omni/engine/arg_utils.py @@ -395,7 +395,7 @@ class OmniAsyncEngineArgs(AsyncEngineArgs, OmniEngineArgs): # Fields in ``SHARED_FIELDS`` (e.g. ``model``, ``log_stats``) flow to BOTH # orchestrator and engine by design. # -# Invariants enforced by ``tests/test_arg_utils.py``: +# Invariants enforced by ``tests/engine/test_arg_utils_shared_fields.py``: # # 1. ``OrchestratorArgs`` ∩ ``OmniEngineArgs`` ⊆ ``SHARED_FIELDS`` # 2. Every CLI flag is classifiable into one of the three buckets From fa0658eabb06baea5e278ef5407e84605c3e03f4 Mon Sep 17 00:00:00 2001 From: Weiming Liao Date: Wed, 29 Jul 2026 23:53:20 +0800 Subject: [PATCH 23/37] =?UTF-8?q?[Doc][NPU]=20Fix=20a=20missed=20version?= =?UTF-8?q?=20reference:=200.25.0=20=E2=86=92=20v0.26.0=20(#5538)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Weiming Liao --- docs/getting_started/installation/npu/npu.inc.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/getting_started/installation/npu/npu.inc.md b/docs/getting_started/installation/npu/npu.inc.md index 19f67cd1d..b24487f3f 100644 --- a/docs/getting_started/installation/npu/npu.inc.md +++ b/docs/getting_started/installation/npu/npu.inc.md @@ -28,7 +28,7 @@ docker run --rm \ -v /etc/ascend_install.info:/etc/ascend_install.info \ -v /root/.cache:/root/.cache \ -p 8000:8000 \ - -it quay.io/atlas-ci/vllm-ascend:0.25.0-a3 bash + -it quay.io/atlas-ci/vllm-ascend:v0.26.0-a3 bash # Inside the container, install vLLM-Omni from source cd /vllm-workspace @@ -122,9 +122,9 @@ Or build IMAGE from **source code**: git clone https://github.com/vllm-project/vllm-omni.git cd vllm-omni # A2 -docker build -t vllm-omni-dev-image:latest -f ./docker/Dockerfile.npu . +# docker build -t vllm-omni-dev-image:latest -f ./docker/Dockerfile.npu . # A3 -# docker build -t vllm-omni-dev-image:latest -f ./docker/Dockerfile.npu.a3 . +docker build -t vllm-omni-dev-image:latest -f ./docker/Dockerfile.npu.a3 . ``` # --8<-- [end:pre-built-images] From f2d9eb163023ab55a7f4cf6e49b024b4f764edf1 Mon Sep 17 00:00:00 2001 From: whisper <131673492+whisper-la@users.noreply.github.com> Date: Thu, 30 Jul 2026 00:49:07 +0800 Subject: [PATCH 24/37] =?UTF-8?q?[Bugfix][CI/Build]=20Avoid=20refetching?= =?UTF-8?q?=20HunyuanVideo=20I2V=20input=20during=20sim=E2=80=A6=20(#5552)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: whisper-la <131673492+whisper-la@users.noreply.github.com> --- .../test_hunyuanvideo15_i2v_video_similarity.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py b/tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py index 84e032770..b830cdc6c 100644 --- a/tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py +++ b/tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py @@ -236,7 +236,8 @@ def test_hunyuanvideo15_i2v_serving_matches_offline_video_similarity( probe_binary("ffmpeg") probe_binary("ffprobe") image_source = _resolve_image_source(hunyuanvideo15_i2v_image_source) - validate_image_source(image_source) + # The input image was already consumed by the prerequisite generation + # tests. Avoid re-fetching a remote source when only comparing artifacts. online_path, offline_path = _artifact_paths(image_source) if not online_path.exists(): pytest.skip(f"Missing online artifact from prerequisite test: {online_path}") From dba05c3517e77b1d3ec85a484ef5286834f8a600 Mon Sep 17 00:00:00 2001 From: zzh <67720593+BLANKETusers@users.noreply.github.com> Date: Thu, 30 Jul 2026 01:21:55 +0800 Subject: [PATCH 25/37] Enable Diffusion Image models asynchronous output (#4981) Signed-off-by: zzh <943967662@qq.com> Co-authored-by: dengyunyang <584797741@qq.com> --- docs/design/feature/async_diffusion_output.md | 174 ++++++++++++ tests/diffusion/test_async_output_worker.py | 193 +++++++++++++ tests/diffusion/test_ipc_async.py | 159 +++++++++++ .../test_multiproc_engine_concurrency.py | 13 +- tests/diffusion/test_result_pump.py | 256 +++++++++++++++++ tests/platforms/test_record_device_event.py | 79 ++++++ vllm_omni/diffusion/data.py | 32 +++ vllm_omni/diffusion/diffusion_engine.py | 15 +- .../diffusion/executor/multiproc_executor.py | 261 ++++++++++++++++-- vllm_omni/diffusion/ipc.py | 83 ++++-- .../hunyuan_image3_transformer.py | 3 +- .../hunyuan_image3/pipeline_hunyuan_image3.py | 43 ++- vllm_omni/diffusion/registry.py | 2 + .../diffusion/sched/request_scheduler.py | 10 +- .../diffusion/worker/diffusion_worker.py | 110 +++++++- vllm_omni/diffusion/worker/utils.py | 1 + vllm_omni/platforms/cuda/platform.py | 11 + vllm_omni/platforms/interface.py | 18 ++ vllm_omni/platforms/musa/platform.py | 10 + vllm_omni/platforms/npu/platform.py | 19 ++ vllm_omni/platforms/rocm/platform.py | 10 + vllm_omni/platforms/xpu/platform.py | 10 + 22 files changed, 1440 insertions(+), 72 deletions(-) create mode 100644 docs/design/feature/async_diffusion_output.md create mode 100644 tests/diffusion/test_async_output_worker.py create mode 100644 tests/diffusion/test_ipc_async.py create mode 100644 tests/diffusion/test_result_pump.py create mode 100644 tests/platforms/test_record_device_event.py diff --git a/docs/design/feature/async_diffusion_output.md b/docs/design/feature/async_diffusion_output.md new file mode 100644 index 000000000..a81ed6811 --- /dev/null +++ b/docs/design/feature/async_diffusion_output.md @@ -0,0 +1,174 @@ +# Async Diffusion Output + +## Table of Contents + +1. [Overview](#overview) +2. [Performance](#performance) +3. [Architecture](#architecture) +4. [Model Coverage](#model-coverage) +5. [Configuration](#configuration) +6. [Related Files](#related-files) + +## Overview + +The async diffusion output feature moves the D2H (Device→Host) copy and SHM packing from the Worker main thread to a background daemon thread, so the GPU default stream can immediately start the next request's forward pass. This eliminates the GPU bubble caused by synchronous D2H/packing. + +**Core benefit**: the previous request's output D2H/packing overlaps with the next request's forward pass. + +**Automatically enabled** when `step_execution=False` (default, request-mode). No extra configuration needed. When `step_execution=True` (step-mode), the original synchronous path is used — pump thread and Worker background thread are not started. + +## Performance + +### HunyuanImage-3.0 (TP4) + +| Resolution | Async Off (QPS) | Async On (QPS) | Change | +|---|---|---|---| +| 1024×1024 | 0.4773 | 0.4802 | +0.60% | +| 768×768 | 0.8370 | 0.8533 | +1.95% | + +**Analysis**: smaller resolution → larger QPS gain. D2H/packing is relatively fixed overhead; forward is variable (scales with resolution). Smaller images → shorter forward → D2H/packing occupies a larger fraction of total time → more benefit from overlapping it. + +## Architecture + +### Execution Timeline + +``` +Before (synchronous D2H): +[forward req1] [D2H+SHM pack req1] [forward req2] [D2H+SHM pack req2] + ^^^^^^^^^^^^^^^^ GPU bubble + +After (async D2H): +[forward req1] [forward req2] +[D2H+SHM pack req1 (side stream)] [D2H+SHM pack req2 (side stream)] + ^^^^^^^^^^^^^^^^ bubble eliminated +``` + +### Data Flow + +``` + ┌─────────────────────────────────────────────────────┐ + │ Worker Process │ + │ │ + execute_model ──> │ WorkerBusyLoop ──> forward() ──> DiffusionOutput │ + │ │ │ (GPU tensor)│ + │ │ ┌─────────┴──────────┐ │ + │ │ │ async output path │ │ + │ │ │ │ │ + │ │ compute_done ──> result_mq │ │ + │ │ │ │ │ + │ │ ▼ │ │ + │ │ AsyncOutputThread │ │ + │ │ (side CUDA stream) │ │ + │ │ wait_event(gpu_event) │ │ + │ │ pack_diffusion_output_shm() │ │ + │ │ D2H + SHM write │ │ + │ │ │ │ │ + │ │ output_ready ──> result_mq │ │ + │ │ │ │ │ + │ ▼ ▼ │ │ + │ (dequeue next request) │ │ + └─────────────────────────────────────────────────────┘ + │ + ▼ + ┌─────────────────────────────────────────────────────┐ + │ Executor (main process) │ + │ │ + │ ResultPumpThread (sole reader of result_mq) │ + │ │ │ + │ ├── AsyncDiffusionOutput(COMPUTE_DONE) │ + │ │ → resolve _rpc_futures[rpc_id] │ + │ │ → RunnerOutput(async_output_id=...) │ + │ │ │ + │ ├── AsyncDiffusionOutput(OUTPUT_READY) │ + │ │ → resolve _output_futures[async_output_id]│ + │ │ → or batch split via _batch_split_map │ + │ │ │ + │ └── Non-async message │ + │ → _sync_result_buffer (for other RPCs) │ + │ │ + │ wait_output_ready(async_output_id) → Future │ + └─────────────────────────────────────────────────────┘ +``` + +### Key Components + +1. **AsyncDiffusionOutput** (`data.py`): Protocol envelope for `result_mq`. `kind` field routes messages: + - `COMPUTE_DONE` — forward finished, GPU can start next request + - `OUTPUT_READY` — D2H/SHM packing finished, final output available + - `RPC_RESULT` — ordinary RPC return (including error propagation) + +2. **Worker AsyncOutputThread** (`diffusion_worker.py`): Background daemon thread that: + - Waits for `gpu_event` (cross-stream sync ensuring forward wrote the tensor) + - Runs `pack_diffusion_output_shm()` on a side CUDA stream + - Enqueues `OUTPUT_READY` to `result_mq` when packing completes + +3. **ResultPumpThread** (`multiproc_executor.py`): Sole reader of `result_mq` in request-mode. Dispatches messages: + - `COMPUTE_DONE` / `RPC_RESULT` → resolves `_rpc_futures[rpc_id]` + - `OUTPUT_READY` → resolves `_output_futures[async_output_id]` (with batch split via `_batch_split_map`) + - Non-async messages → `_sync_result_buffer` for other RPCs + +4. **collective_rpc() two-path dispatch** (`multiproc_executor.py`): + - **Path 1**: `execute_model` / `execute_model_batch` → generates `rpc_id`, registers Future, waits for pump to deliver `compute_done` + - **Path 2**: All other RPCs → `_sync_result_buffer` (pump-fed) or `_result_mq` (step-mode, no pump) + +5. **record_device_event** (`platforms/`): Cross-platform GPU event recording for side-stream synchronization. Implemented for CUDA, NPU, ROCm, XPU, MUSA; base class returns `None` (safe no-op). + +6. **IPC side-stream D2H** (`ipc.py`): `pack_diffusion_output_shm()` and helpers accept `d2h_stream` parameter; use `pin_memory` + `copy_(non_blocking=True)` on the side stream instead of synchronous `.cpu()`. + +### Batch Split + +When `execute_model_batch` returns a `COMPUTE_DONE` with a single `async_output_id` for the entire batch, the executor splits it into per-request `async_output_id`s (format `{batch_id}/{request_id}`) via `_batch_split_map`. When `OUTPUT_READY` arrives, the pump extracts per-request results from the batch output and resolves each request's output future independently. + +## Model Coverage + +### Currently Supported (request-mode, `step_execution=False`) + +All models running in request-mode (`step_execution=False`, the default) automatically use async output. + +**Verified models**: + +| Model | Type | `supports_request_batch` | +|---|---|---| +| **HunyuanImage-3.0** | Image | `False` | +| **Qwen-Image** | Image | `True` | + +Other models with `step_execution=False` are also supported but not yet verified. + +### Not Yet Supported (step-mode, `step_execution=True`) + +When `step_execution=True` (or `streaming_output=True`, which auto-enables step-mode), models use `execute_stepwise` instead of `execute_model`, which is not in the async Path 1 whitelist. Async output is not applicable. + +**Verified**: Helios (`step_execution=True`, does not benefit from this feature). + +Other models with `step_execution=True` are also not applicable. + +Adapting step-mode requires additional design because: + +- Each step's `RunnerOutput` must be available synchronously for the scheduler to advance +- Intermediate latent tensors may not need D2H (only the final step needs D2H + postprocess) +- Steps are serial (step N+1 depends on step N output), so the overlap benefit is limited to "final step's D2H overlaps with the next request's forward" + +**Planned approach**: first cover only the final step's async D2H, keep intermediate steps synchronous. This matches the request-mode benefit pattern. + +## Configuration + +No configuration needed. Async output is automatically enabled when `step_execution=False` (default). + +| `step_execution` | Mode | Async Output | Pump Thread | Worker BG Thread | +|---|---|---|---|---| +| `False` (default) | request-mode | ✅ enabled | ✅ started | ✅ started | +| `True` | step-mode | ❌ disabled | ❌ not started | ❌ not started | + +## Related Files + +- `vllm_omni/diffusion/data.py`: `AsyncDiffusionOutput`, `AsyncOutputKind`, `DiffusionOutput.async_output_id` +- `vllm_omni/diffusion/worker/diffusion_worker.py`: `WorkerProc._return_result()`, `_async_output_loop()`, `_generate_async_output_id()` +- `vllm_omni/diffusion/executor/multiproc_executor.py`: `ResultPumpThread`, `collective_rpc()` two-path dispatch, `wait_output_ready()`, `_batch_split_map` +- `vllm_omni/diffusion/diffusion_engine.py`: `step()` / `step_streaming()` / `add_req_and_wait_for_response()` async output waiting +- `vllm_omni/diffusion/sched/request_scheduler.py`: `update_from_output()` — `async_output_id` → `FINISHED_COMPLETED` +- `vllm_omni/diffusion/ipc.py`: `pack_diffusion_output_shm()` side-stream D2H path +- `vllm_omni/platforms/interface.py`: `OmniPlatform.record_device_event()` base class +- `vllm_omni/platforms/cuda/platform.py`: `CudaOmniPlatform.record_device_event()` +- `vllm_omni/platforms/npu/platform.py`: `NPUOmniPlatform.record_device_event()` +- `vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py`: `get_hunyuan_image3_post_process_func()`, postprocess removal +- `vllm_omni/diffusion/registry.py`: `_DIFFUSION_POST_PROCESS_FUNCS` postprocess registry diff --git a/tests/diffusion/test_async_output_worker.py b/tests/diffusion/test_async_output_worker.py new file mode 100644 index 000000000..8cda3be1c --- /dev/null +++ b/tests/diffusion/test_async_output_worker.py @@ -0,0 +1,193 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for WorkerProc async output logic.""" + +import queue +from unittest.mock import MagicMock + +import pytest + +from vllm_omni.diffusion.data import ( + AsyncDiffusionOutput, + AsyncOutputKind, + DiffusionOutput, + OmniACK, +) +from vllm_omni.diffusion.worker.diffusion_worker import WorkerProc +from vllm_omni.diffusion.worker.utils import BatchRunnerOutput, RunnerOutput + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _make_worker_proc(step_execution=False): + """Create a minimal WorkerProc-like object with async output state.""" + proc = object.__new__(WorkerProc) + proc.od_config = MagicMock() + proc.od_config.step_execution = step_execution + proc.gpu_id = 0 + proc.result_mq = MagicMock() + proc._running = True + proc._async_output_queue = queue.Queue() if not step_execution else None + proc._async_output_thread = None + return proc + + +class TestGenerateAsyncOutputId: + """Test _generate_async_output_id uniqueness.""" + + def test_returns_hex_string(self): + id1 = WorkerProc._generate_async_output_id() + assert isinstance(id1, str) + assert len(id1) == 32 # uuid4 hex is 32 chars + + def test_ids_are_unique(self): + ids = {WorkerProc._generate_async_output_id() for _ in range(100)} + assert len(ids) == 100 + + +class TestReturnResultAsyncPath: + """Test _return_result in request-mode (step_execution=False).""" + + def test_omni_ack_bypasses_async_path(self): + """OmniACK should always use sync path regardless of step_execution.""" + proc = _make_worker_proc(step_execution=False) + ack = OmniACK(task_id="task1", status="done") + + proc._return_result(ack) + proc.result_mq.enqueue.assert_called_once_with(ack) + assert proc._async_output_queue.empty() # nothing enqueued for bg thread + + def test_diffusion_output_triggers_async_path(self, mocker): + """DiffusionOutput in request-mode should enqueue compute_done + bg work.""" + proc = _make_worker_proc(step_execution=False) + + mock_platform = mocker.MagicMock() + mock_platform.record_device_event.return_value = MagicMock() + mocker.patch( + "vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform", + mock_platform, + ) + + output = DiffusionOutput(output="data") + proc._return_result(output, rpc_id="1") + + # Should have enqueued COMPUTE_DONE to result_mq + enqueued_msg = proc.result_mq.enqueue.call_args[0][0] + assert isinstance(enqueued_msg, AsyncDiffusionOutput) + assert enqueued_msg.kind == AsyncOutputKind.COMPUTE_DONE + assert enqueued_msg.rpc_id == "1" + assert enqueued_msg.async_output_id is not None + + # Should have placed work item in async_output_queue + assert not proc._async_output_queue.empty() + item = proc._async_output_queue.get_nowait() + assert item[0] is output + assert item[1] == enqueued_msg.async_output_id + + def test_batch_runner_output_triggers_async_path(self, mocker): + """BatchRunnerOutput in request-mode should also go through async path.""" + proc = _make_worker_proc(step_execution=False) + + mock_platform = mocker.MagicMock() + mock_platform.record_device_event.return_value = MagicMock() + mocker.patch( + "vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform", + mock_platform, + ) + + batch = BatchRunnerOutput( + runner_outputs=[RunnerOutput(request_id="r1", finished=True)], + ) + proc._return_result(batch) + + enqueued_msg = proc.result_mq.enqueue.call_args[0][0] + assert isinstance(enqueued_msg, AsyncDiffusionOutput) + assert enqueued_msg.kind == AsyncOutputKind.COMPUTE_DONE + + def test_none_result_mq_skips_all_paths(self): + """If result_mq is None, _return_result should do nothing.""" + proc = _make_worker_proc(step_execution=False) + proc.result_mq = None + + proc._return_result(DiffusionOutput()) + # No crash, no enqueue attempted + + +class TestReturnResultSyncPath: + """Test _return_result in step-execution mode (sync).""" + + def test_diffusion_output_uses_sync_path_when_step_execution(self, mocker): + """In step_execution mode, DiffusionOutput goes through sync SHM pack.""" + proc = _make_worker_proc(step_execution=True) + proc._async_output_queue = None # step mode doesn't have async queue + + mock_pack = mocker.patch( + "vllm_omni.diffusion.worker.diffusion_worker.pack_diffusion_output_shm", + ) + output = DiffusionOutput(output="data") + proc._return_result(output) + + mock_pack.assert_called_once_with(output) + proc.result_mq.enqueue.assert_called_once_with(output) + + def test_non_diffusion_output_skips_async_in_request_mode(self, mocker): + """Non-DiffusionOutput/BatchRunnerOutput in request mode uses sync path.""" + proc = _make_worker_proc(step_execution=False) + + mock_pack = mocker.patch( + "vllm_omni.diffusion.worker.diffusion_worker.pack_diffusion_output_shm", + ) + # A plain dict is neither DiffusionOutput nor BatchRunnerOutput + output = {"type": "other"} + proc._return_result(output) + + mock_pack.assert_called_once_with(output) + proc.result_mq.enqueue.assert_called_once_with(output) + + +class TestAsyncOutputLoopLogic: + """Test _async_output_loop background thread logic (mocked, no device).""" + + def test_bg_thread_enqueues_output_ready_after_pack(self, mocker): + """The real _async_output_loop packs via SHM with d2h_stream and + enqueues an OUTPUT_READY message.""" + import threading + import time + + proc = _make_worker_proc(step_execution=False) + + mock_pack = mocker.patch( + "vllm_omni.diffusion.worker.diffusion_worker.pack_diffusion_output_shm", + ) + # Mock torch to avoid device initialization + mock_torch = mocker.MagicMock() + mock_stream = mocker.MagicMock() + mock_torch.Stream.return_value = mock_stream + mock_torch.accelerator.current_accelerator.return_value.type = "cpu" + mock_torch.device.return_value = mocker.MagicMock() + mocker.patch( + "vllm_omni.diffusion.worker.diffusion_worker.torch", + mock_torch, + ) + + output = DiffusionOutput(output="data") + gpu_event = mocker.MagicMock() + proc._async_output_queue.put((output, "abc123", gpu_event)) + + # Run the real _async_output_loop in a background thread. + t = threading.Thread(target=proc._async_output_loop, daemon=True) + t.start() + + # Give the loop time to process, then unblock and stop. + time.sleep(0.5) + proc._running = False + proc._async_output_queue.put((DiffusionOutput(), "sentinel", None)) + t.join(timeout=2.0) + + mock_pack.assert_called() + # Verify an OUTPUT_READY with the correct async_output_id was enqueued. + output_ready_enqueued = any( + c[0][0].kind == AsyncOutputKind.OUTPUT_READY and c[0][0].async_output_id == "abc123" + for c in proc.result_mq.enqueue.call_args_list + ) + assert output_ready_enqueued, "Expected OUTPUT_READY with async_output_id='abc123' in enqueue calls" diff --git a/tests/diffusion/test_ipc_async.py b/tests/diffusion/test_ipc_async.py new file mode 100644 index 000000000..c54d2f793 --- /dev/null +++ b/tests/diffusion/test_ipc_async.py @@ -0,0 +1,159 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for IPC async D2H stream packing.""" + +import pytest +import torch + +from vllm_omni.diffusion.data import DiffusionOutput +from vllm_omni.diffusion.ipc import ( + _pack_diffusion_fields, + _pack_tensor_if_large, + _pack_value_if_large, + pack_diffusion_output_shm, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class TestPackTensorIfLargeWithD2hStream: + """Test _pack_tensor_if_large with mock d2h_stream parameter.""" + + def test_small_tensor_not_packed_with_stream(self, mocker): + """Small tensors pass through even when d2h_stream is provided.""" + small_tensor = torch.zeros(10) # way below threshold + mock_stream = mocker.MagicMock() + result = _pack_tensor_if_large(small_tensor, d2h_stream=mock_stream) + # Small tensor should be returned as-is (not a SHM handle dict) + assert isinstance(result, torch.Tensor) + + def test_large_tensor_packed_without_stream(self, mocker): + """Large tensor is packed via SHM when no d2h_stream.""" + large_tensor = torch.randn(500_000) # > 1MB for float32 + result = _pack_tensor_if_large(large_tensor, d2h_stream=None) + assert isinstance(result, dict) + assert result.get("__tensor_shm__") is True + + def test_pack_value_if_large_passes_stream_to_tensor_pack(self, mocker): + """_pack_value_if_large propagates d2h_stream to _pack_tensor_if_large.""" + mock_pack = mocker.patch( + "vllm_omni.diffusion.ipc._pack_tensor_if_large", + return_value="packed", + ) + tensor = torch.zeros(1) + d2h = mocker.MagicMock() + _pack_value_if_large(tensor, d2h_stream=d2h) + mock_pack.assert_called_once_with(tensor, d2h_stream=d2h) + + def test_pack_value_if_large_passes_stream_through_dict(self, mocker): + """d2h_stream propagates through dict container packing.""" + mock_pack = mocker.patch( + "vllm_omni.diffusion.ipc._pack_tensor_if_large", + return_value="packed", + ) + d2h = mocker.MagicMock() + data = {"image": torch.zeros(1), "video": torch.zeros(1)} + _pack_value_if_large(data, d2h_stream=d2h) + assert mock_pack.call_count == 2 + for call in mock_pack.call_args_list: + assert call.kwargs.get("d2h_stream") is d2h or call.args[1] is d2h + + def test_pack_value_if_large_passes_stream_through_list(self, mocker): + """d2h_stream propagates through list container packing.""" + mock_pack = mocker.patch( + "vllm_omni.diffusion.ipc._pack_tensor_if_large", + return_value="packed", + ) + d2h = mocker.MagicMock() + data = [torch.zeros(1), torch.zeros(1)] + _pack_value_if_large(data, d2h_stream=d2h) + assert mock_pack.call_count == 2 + + def test_pack_value_if_large_passes_stream_through_tuple(self, mocker): + """d2h_stream propagates through tuple container packing.""" + mock_pack = mocker.patch( + "vllm_omni.diffusion.ipc._pack_tensor_if_large", + return_value="packed", + ) + d2h = mocker.MagicMock() + data = (torch.zeros(1), torch.zeros(1)) + result = _pack_value_if_large(data, d2h_stream=d2h) + assert isinstance(result, tuple) + assert mock_pack.call_count == 2 + + +class TestPackDiffusionFieldsWithD2hStream: + """Test _pack_diffusion_fields propagates d2h_stream.""" + + def test_output_field_packed_with_stream(self, mocker): + mock_pack_value = mocker.patch( + "vllm_omni.diffusion.ipc._pack_value_if_large", + return_value="packed", + ) + d2h = mocker.MagicMock() + out = DiffusionOutput(output=torch.zeros(1)) + _pack_diffusion_fields(out, d2h_stream=d2h) + mock_pack_value.assert_called_once_with(torch.zeros(1), d2h_stream=d2h) + + def test_trajectory_fields_packed_with_stream(self, mocker): + mock_pack_tensor = mocker.patch( + "vllm_omni.diffusion.ipc._pack_tensor_if_large", + return_value="packed", + ) + d2h = mocker.MagicMock() + out = DiffusionOutput( + trajectory_latents=torch.zeros(1), + trajectory_log_probs=torch.zeros(1), + ) + _pack_diffusion_fields(out, d2h_stream=d2h) + # Should pack trajectory_latents and trajectory_log_probs + assert mock_pack_tensor.call_count == 2 + + +class TestPackDiffusionOutputShmWithD2hStream: + """Test pack_diffusion_output_shm top-level dispatch with d2h_stream.""" + + def test_diffusion_output_path_with_stream(self, mocker): + mock_pack_fields = mocker.patch( + "vllm_omni.diffusion.ipc._pack_diffusion_fields", + ) + d2h = mocker.MagicMock() + out = DiffusionOutput() + pack_diffusion_output_shm(out, d2h_stream=d2h) + mock_pack_fields.assert_called_once_with(out, d2h_stream=d2h) + + def test_rpc_envelope_path_with_stream(self, mocker): + mock_pack_fields = mocker.patch( + "vllm_omni.diffusion.ipc._pack_diffusion_fields", + ) + d2h = mocker.MagicMock() + result = DiffusionOutput() + envelope = {"type": "diffusion_rpc_result", "result": result} + pack_diffusion_output_shm(envelope, d2h_stream=d2h) + # envelope["result"] is replaced in-place by pack_diffusion_output_shm; + # assert with the original reference, not the replaced one. + mock_pack_fields.assert_called_once_with( + result, + d2h_stream=d2h, + ) + + def test_runner_output_path_with_stream(self, mocker): + mock_pack_fields = mocker.patch( + "vllm_omni.diffusion.ipc._pack_diffusion_fields", + ) + d2h = mocker.MagicMock() + + from vllm_omni.diffusion.worker.utils import RunnerOutput + + result = DiffusionOutput() + runner_out = RunnerOutput( + request_id="req1", + result=result, + ) + pack_diffusion_output_shm(runner_out, d2h_stream=d2h) + # runner_out.result is replaced in-place by pack_diffusion_output_shm; + # assert with the original reference, not the replaced one. + mock_pack_fields.assert_called_once_with( + result, + d2h_stream=d2h, + ) diff --git a/tests/diffusion/test_multiproc_engine_concurrency.py b/tests/diffusion/test_multiproc_engine_concurrency.py index f78f8d44b..10fe7e86c 100644 --- a/tests/diffusion/test_multiproc_engine_concurrency.py +++ b/tests/diffusion/test_multiproc_engine_concurrency.py @@ -56,7 +56,7 @@ def _make_executor(num_gpus: int = 1): Returns ``(executor, request_queue, result_queue)``. """ - od_cfg = SimpleNamespace(num_gpus=num_gpus, streaming_output=False) + od_cfg = SimpleNamespace(num_gpus=num_gpus, streaming_output=False, step_execution=True) executor = object.__new__(MultiprocDiffusionExecutor) executor.od_config = od_cfg @@ -616,6 +616,7 @@ class TestMultiprocExecutorRaisesEngineDeadError: def test_collective_rpc_raises_when_is_failed(self): executor = object.__new__(MultiprocDiffusionExecutor) + executor.od_config = SimpleNamespace(step_execution=True) executor._closed = False executor._broadcast_mq = MagicMock() executor._result_mq = MagicMock() @@ -661,7 +662,7 @@ class TestMultiprocExecutorStepStreamingOutput: def test_execute_step_allows_streaming_output_mode(self): executor, req_q, res_q = _make_executor() - executor.od_config = SimpleNamespace(streaming_output=True) # pyright: ignore[reportAttributeAccessIssue] + executor.od_config = SimpleNamespace(streaming_output=True, step_execution=True) # pyright: ignore[reportAttributeAccessIssue] runner_outputs = [ RunnerOutput( request_id="sched-stream", @@ -939,6 +940,14 @@ class TestMultiprocExecutorWorkerMonitor: executor._shutdown_cleaner = None # Use a no-op so shutdown() doesn't crash on None resources. executor._finalizer = lambda: None + # ------------------------------------------------------------------ + # Attributes added by remove_bubble_v2 (async D2H); shutdown() iterates + # over them, so they need to exist even when constructed via __new__. + executor._pump_stop = threading.Event() + executor._futures_lock = threading.RLock() + executor._rpc_futures = {} + executor._output_futures = {} + executor._batch_split_map = {} proc = _make_short_lived_process() executor._processes = [proc] diff --git a/tests/diffusion/test_result_pump.py b/tests/diffusion/test_result_pump.py new file mode 100644 index 000000000..fe49df47e --- /dev/null +++ b/tests/diffusion/test_result_pump.py @@ -0,0 +1,256 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for MultiprocDiffusionExecutor async result pump and wait_output_ready.""" + +import concurrent.futures +import queue +import threading +import time +from unittest.mock import MagicMock + +import pytest + +from vllm_omni.diffusion.data import AsyncDiffusionOutput, AsyncOutputKind, DiffusionOutput + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _make_executor(step_execution=False): + """Create a minimal MultiprocDiffusionExecutor-like object with pump state.""" + from vllm_omni.diffusion.executor.multiproc_executor import MultiprocDiffusionExecutor + + od_config = MagicMock() + od_config.step_execution = step_execution + + executor = object.__new__(MultiprocDiffusionExecutor) + executor.od_config = od_config + executor._rpc_id_counter = 0 + executor._rpc_id_lock = threading.Lock() + executor._rpc_futures = {} + executor._output_futures = {} + executor._completed_outputs = {} + executor._batch_split_map = {} + executor._futures_lock = threading.RLock() + executor._pump_running = False + executor._pump_stop = threading.Event() + executor._sync_result_buffer = queue.Queue() + executor._result_mq = MagicMock() + executor._broadcast_mq = MagicMock() + executor._closed = False + executor._is_failed = False + executor._finalizer = MagicMock() # no-op in tests + executor._shutdown_cleaner = None + executor._processes = [] + return executor + + +def _feed_one_msg_to_pump(executor, msg): + """Run _result_pump in a daemon thread, feed one *msg*, then stop.""" + call_count = [0] + + def mock_dequeue(timeout=None): + call_count[0] += 1 + if call_count[0] == 1: + return msg + executor._pump_stop.set() + time.sleep(0.05) + raise TimeoutError + + executor._result_mq.dequeue = mock_dequeue + t = threading.Thread(target=executor._result_pump, daemon=True) + t.start() + t.join(timeout=2.0) + + +@pytest.fixture(autouse=True) +def _mock_unpack(mocker): + """Real _result_pump calls unpack_diffusion_output_shm; mock it away.""" + mocker.patch( + "vllm_omni.diffusion.executor.multiproc_executor.unpack_diffusion_output_shm", + ) + + +class TestNextRpcId: + """Test _next_rpc_id counter.""" + + def test_counter_increments(self): + executor = _make_executor() + id1 = executor._next_rpc_id() + id2 = executor._next_rpc_id() + id3 = executor._next_rpc_id() + assert id1 == "1" + assert id2 == "2" + assert id3 == "3" + + def test_counter_is_threadsafe(self): + executor = _make_executor() + ids = [] + + def get_id(): + ids.append(executor._next_rpc_id()) + + threads = [threading.Thread(target=get_id) for _ in range(10)] + for t in threads: + t.start() + for t in threads: + t.join() + + # All IDs should be unique + assert len(set(ids)) == 10 + + +class TestWaitOutputReady: + """Test wait_output_ready future creation and caching.""" + + def test_returns_new_future_when_not_cached(self): + executor = _make_executor() + fut = executor.wait_output_ready("abc123") + assert isinstance(fut, concurrent.futures.Future) + assert not fut.done() + + def test_future_resolves_when_output_arrives(self): + executor = _make_executor() + fut = executor.wait_output_ready("abc123") + output = DiffusionOutput(output="data") + + # Simulate pump resolving the future + with executor._futures_lock: + executor._output_futures.pop("abc123") + if not fut.done(): + fut.set_result(output) + + assert fut.result(timeout=1.0) is output + + def test_returns_cached_future_when_already_completed(self): + executor = _make_executor() + output = DiffusionOutput(output="cached_data") + fut = concurrent.futures.Future() + fut.set_result(output) + with executor._futures_lock: + executor._completed_outputs["abc123"] = fut + + fut = executor.wait_output_ready("abc123") + assert fut.done() + assert fut.result(timeout=1.0) is output + + def test_removes_from_cache_after_retrieval(self): + executor = _make_executor() + output = DiffusionOutput(output="cached_data") + with executor._futures_lock: + executor._completed_outputs["abc123"] = output + + executor.wait_output_ready("abc123") + # Second call should not find cached result + with executor._futures_lock: + assert "abc123" not in executor._completed_outputs + + +class TestResultPumpDispatch: + """Test _result_pump message routing (running the real pump in a thread).""" + + def test_non_async_message_placed_in_sync_buffer(self): + executor = _make_executor() + msg = DiffusionOutput(output="sync_result") + _feed_one_msg_to_pump(executor, msg) + + assert not executor._sync_result_buffer.empty() + retrieved = executor._sync_result_buffer.get_nowait() + assert isinstance(retrieved, DiffusionOutput) + + def test_compute_done_routes_to_rpc_future(self): + executor = _make_executor() + rpc_id = "42" + fut = concurrent.futures.Future() + with executor._futures_lock: + executor._rpc_futures[rpc_id] = fut + + msg = AsyncDiffusionOutput( + kind=AsyncOutputKind.COMPUTE_DONE, + rpc_id=rpc_id, + async_output_id="abc", + ) + _feed_one_msg_to_pump(executor, msg) + + assert fut.done() + result = fut.result(timeout=1.0) + assert result.kind == AsyncOutputKind.COMPUTE_DONE + + def test_output_ready_routes_to_output_future(self): + executor = _make_executor() + async_output_id = "abc123" + output = DiffusionOutput(output="final") + fut = concurrent.futures.Future() + with executor._futures_lock: + executor._output_futures[async_output_id] = fut + + msg = AsyncDiffusionOutput( + kind=AsyncOutputKind.OUTPUT_READY, + async_output_id=async_output_id, + output=output, + ) + _feed_one_msg_to_pump(executor, msg) + + assert fut.done() + assert fut.result(timeout=1.0) is output + + def test_output_ready_with_error_routes_to_future_as_exception(self): + executor = _make_executor() + async_output_id = "abc123" + fut = concurrent.futures.Future() + with executor._futures_lock: + executor._output_futures[async_output_id] = fut + + msg = AsyncDiffusionOutput( + kind=AsyncOutputKind.OUTPUT_READY, + async_output_id=async_output_id, + error="Background D2H/SHM packing failed", + ) + _feed_one_msg_to_pump(executor, msg) + + assert fut.done() + with pytest.raises(RuntimeError, match="Background D2H/SHM packing failed"): + fut.result(timeout=1.0) + + def test_output_ready_caches_when_no_future_waiting(self): + """When OUTPUT_READY arrives but no future is waiting, result is cached.""" + executor = _make_executor() + async_output_id = "abc123" + output = DiffusionOutput(output="orphan") + + msg = AsyncDiffusionOutput( + kind=AsyncOutputKind.OUTPUT_READY, + async_output_id=async_output_id, + output=output, + ) + _feed_one_msg_to_pump(executor, msg) + + # Later call to wait_output_ready should find it cached + fut = executor.wait_output_ready(async_output_id) + assert fut.done() + assert fut.result(timeout=1.0) is output + + +class TestShutdownCleansUpFutures: + """Test that shutdown cancels pending futures.""" + + def test_shutdown_sets_exception_on_pending_futures(self): + executor = _make_executor() + + rpc_fut = concurrent.futures.Future() + output_fut = concurrent.futures.Future() + with executor._futures_lock: + executor._rpc_futures["1"] = rpc_fut + executor._output_futures["abc"] = output_fut + + executor.shutdown() + + assert rpc_fut.done() + with pytest.raises(RuntimeError, match="Executor shut down"): + rpc_fut.result(timeout=1.0) + + assert output_fut.done() + with pytest.raises(RuntimeError, match="Executor shut down"): + output_fut.result(timeout=1.0) + + assert len(executor._rpc_futures) == 0 + assert len(executor._output_futures) == 0 diff --git a/tests/platforms/test_record_device_event.py b/tests/platforms/test_record_device_event.py new file mode 100644 index 000000000..a979ca940 --- /dev/null +++ b/tests/platforms/test_record_device_event.py @@ -0,0 +1,79 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for OmniPlatform.record_device_event implementations.""" + +import pytest + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class TestOmniPlatformRecordDeviceEventInterface: + """Test that the base OmniPlatform returns None (safe no-op fallback).""" + + def test_base_class_returns_none(self): + from vllm_omni.platforms.interface import OmniPlatform + + assert OmniPlatform.record_device_event() is None + + +class TestCudaOmniPlatformRecordDeviceEvent: + """Test CudaOmniPlatform.record_device_event with mocked torch.Event.""" + + def test_records_event_successfully(self, mocker): + from vllm_omni.platforms.cuda.platform import CudaOmniPlatform + + mock_event = mocker.MagicMock() + mocker.patch("torch.Event", return_value=mock_event) + + result = CudaOmniPlatform.record_device_event() + assert result is mock_event + mock_event.record.assert_called_once() + + def test_returns_none_on_failure(self, mocker): + from vllm_omni.platforms.cuda.platform import CudaOmniPlatform + + mocker.patch("torch.Event", side_effect=RuntimeError("no CUDA")) + + result = CudaOmniPlatform.record_device_event() + assert result is None + + +class TestNPUOmniPlatformRecordDeviceEvent: + """Test NPUOmniPlatform.record_device_event with mocked torch.npu.""" + + def test_returns_none_on_failure(self, mocker): + """When torch.npu.current_stream().synchronize() fails, return None.""" + try: + from vllm_omni.platforms.npu.platform import NPUOmniPlatform + except ModuleNotFoundError: + pytest.skip("vllm_ascend not available") + + mock_torch = mocker.MagicMock() + mock_torch.npu.current_stream.return_value.synchronize.side_effect = RuntimeError("no NPU") + mocker.patch("vllm_omni.platforms.npu.platform.torch", mock_torch) + + result = NPUOmniPlatform.record_device_event() + assert result is None + + def test_synchronizes_stream_then_records_event(self, mocker): + """NPU should sync stream first, then create and record an Event.""" + try: + from vllm_omni.platforms.npu.platform import NPUOmniPlatform + except ModuleNotFoundError: + pytest.skip("vllm_ascend not available") + + mock_event = mocker.MagicMock() + mock_stream = mocker.MagicMock() + mock_torch = mocker.MagicMock() + mock_torch.npu.current_stream.return_value = mock_stream + mock_torch.npu.Event.return_value = mock_event + mocker.patch("vllm_omni.platforms.npu.platform.torch", mock_torch) + + result = NPUOmniPlatform.record_device_event() + + # Stream should be synced first (HCCL ordering) + mock_stream.synchronize.assert_called_once() + # Then event should be created and recorded + mock_torch.npu.Event.assert_called_once() + mock_event.record.assert_called_once() + assert result is mock_event diff --git a/vllm_omni/diffusion/data.py b/vllm_omni/diffusion/data.py index d3880f900..ca9d48d96 100644 --- a/vllm_omni/diffusion/data.py +++ b/vllm_omni/diffusion/data.py @@ -1273,6 +1273,7 @@ class DiffusionOutput: trajectory_latents: torch.Tensor | dict[str, Any] | None = None trajectory_log_probs: torch.Tensor | dict[str, Any] | None = None trajectory_decoded: list[Image.Image] | None = None + async_output_id: str | None = None error: str | None = None error_status_code: int | None = None error_type: str | None = None @@ -1333,6 +1334,37 @@ class DiffusionOutput: ) +class AsyncOutputKind(Enum): + """Message kind for ``AsyncDiffusionOutput`` — routes async results. + + * ``RPC_RESULT`` — ordinary RPC return (sleep, wake, profile, etc.) + * ``COMPUTE_DONE`` — worker forward finished, GPU can start next request + * ``OUTPUT_READY`` — background D2H/SHM packing finished, final output + is available via ``async_output_id`` + """ + + RPC_RESULT = "rpc_result" + COMPUTE_DONE = "compute_done" + OUTPUT_READY = "output_ready" + + +@dataclass +class AsyncDiffusionOutput: + """Async protocol envelope for ``result_mq`` messages. + + In request-mode (``step_execution=False``), all ``result_mq`` + messages use this envelope. The ``kind`` field routes the message + to the correct consumer. + """ + + kind: AsyncOutputKind + rpc_id: str | None = None + async_output_id: str | None = None + result: Any | None = None + output: DiffusionOutput | None = None + error: str | None = None + + class DiffusionRequestAbortedError(RuntimeError): """Raised when a diffusion request ends via user-visible abort.""" diff --git a/vllm_omni/diffusion/diffusion_engine.py b/vllm_omni/diffusion/diffusion_engine.py index 4d37843d7..44ca9da89 100644 --- a/vllm_omni/diffusion/diffusion_engine.py +++ b/vllm_omni/diffusion/diffusion_engine.py @@ -55,6 +55,8 @@ if TYPE_CHECKING: logger = init_logger(__name__) +_ASYNC_OUTPUT_TIMEOUT = 30.0 # seconds + __all__ = [ "DiffusionEngine", "DiffusionExecutionMode", @@ -288,6 +290,10 @@ class DiffusionEngine: generator = self.async_add_req_and_stream_response(request) async for output in generator: exec_total_time = time.perf_counter() - exec_start_time + # Async mode: wait for background D2H/SHM to complete. + if output.async_output_id: + fut = self.executor.wait_output_ready(output.async_output_id) + output = await asyncio.wait_for(asyncio.wrap_future(fut), timeout=_ASYNC_OUTPUT_TIMEOUT) postprocess_start_time = time.perf_counter() formatted_outputs = self.postprocess_output(request, output) postprocess_time = time.perf_counter() - postprocess_start_time @@ -744,11 +750,15 @@ class DiffusionEngine: raise ValueError("Sync func should receive one result at one time") if target_request_id in finished_req_ids: req_output = runner_output.get_request_output(target_request_id) - return self._finalize_finished_request( + output = self._finalize_finished_request( target_request_id, runner_output=req_output, missing_result_error="Diffusion execution finished without a final output.", ) + if output.async_output_id: + fut = self.executor.wait_output_ready(output.async_output_id) + output = fut.result(timeout=_ASYNC_OUTPUT_TIMEOUT) + return output def profile(self, is_start: bool = True, profile_prefix: str | None = None) -> None: """Start or stop profiling on all diffusion workers. @@ -1036,4 +1046,7 @@ class DiffusionEngine: if runner_output is not None and runner_output.result is not None: return runner_output.result + if runner_output is not None and runner_output.async_output_id is not None: + return DiffusionOutput(async_output_id=runner_output.async_output_id) + return DiffusionOutput(error=missing_result_error) diff --git a/vllm_omni/diffusion/executor/multiproc_executor.py b/vllm_omni/diffusion/executor/multiproc_executor.py index c7ca34153..84a460c53 100644 --- a/vllm_omni/diffusion/executor/multiproc_executor.py +++ b/vllm_omni/diffusion/executor/multiproc_executor.py @@ -1,7 +1,9 @@ from __future__ import annotations +import concurrent.futures import multiprocessing as mp import multiprocessing.connection +import queue import threading import time import weakref @@ -15,7 +17,7 @@ from vllm.distributed.device_communicators.shm_broadcast import Handle, MessageQ from vllm.logger import init_logger from vllm.v1.engine.exceptions import EngineDeadError -from vllm_omni.diffusion.data import SHUTDOWN_MESSAGE, DiffusionOutput +from vllm_omni.diffusion.data import SHUTDOWN_MESSAGE, AsyncDiffusionOutput, AsyncOutputKind, DiffusionOutput from vllm_omni.diffusion.executor.abstract import DiffusionExecutor from vllm_omni.diffusion.ipc import DIFFUSION_RPC_RESULT_ENVELOPE, unpack_diffusion_output_shm from vllm_omni.diffusion.worker import WorkerProc @@ -93,6 +95,23 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): self.shutdown() raise + # Async output: result pump thread for dispatching AsyncDiffusionOutput + # messages in request-mode (step_execution=False). + self._rpc_id_counter = 0 + self._rpc_id_lock = threading.Lock() + self._rpc_futures: dict[str, concurrent.futures.Future[AsyncDiffusionOutput]] = {} + self._output_futures: dict[str, concurrent.futures.Future[DiffusionOutput]] = {} + self._completed_outputs: dict[str, concurrent.futures.Future[DiffusionOutput]] = {} + self._batch_split_map: dict[str, dict[str, str]] = {} # batch_id -> {per_req_id: request_id} + self._futures_lock = threading.RLock() + self._pump_running = False + self._pump_stop = threading.Event() + # When pump is active it is the sole reader of result_mq; non-async + # messages are placed here for collective_rpc() to consume. + self._sync_result_buffer: queue.Queue = queue.Queue() + if not self.od_config.step_execution: + self._start_result_pump() + self._start_worker_monitor() def _init_broadcast_queue(self, num_workers: int) -> MessageQueue: @@ -116,7 +135,12 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): raise RuntimeError("Broadcast queue is closed") def _dequeue_one_with_failure_polling(self, deadline: float | None, method: str) -> Any: - """Block until one result message, polling ``_is_failed`` between chunk timeouts.""" + """Block until one result message, polling ``_is_failed`` between chunk timeouts. + + When async output is enabled, pump is the sole reader of result_mq; + non-async messages are placed in _sync_result_buffer, so this method + reads from there instead of result_mq directly. + """ while True: if deadline is None: chunk_timeout = _DEQUEUE_TIMEOUT_S @@ -125,12 +149,20 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): if remaining <= 0: raise TimeoutError(f"RPC call to {method} timed out.") chunk_timeout = min(_DEQUEUE_TIMEOUT_S, remaining) - try: - return self._result_mq.dequeue(timeout=chunk_timeout) # pyright: ignore[reportOptionalMemberAccess] MQ is not None before shutdown - except (TimeoutError, zmq.error.Again): - if self._is_failed: - raise EngineDeadError() - continue + if not self.od_config.step_execution: + try: + return self._sync_result_buffer.get(timeout=chunk_timeout) + except queue.Empty: + if self._is_failed or self._closed: + raise EngineDeadError() + continue + else: + try: + return self._result_mq.dequeue(timeout=chunk_timeout) # pyright: ignore[reportOptionalMemberAccess] MQ is not None before shutdown + except (TimeoutError, zmq.error.Again): + if self._is_failed: + raise EngineDeadError() + continue @staticmethod def _raise_for_rpc_error_dict(response: Any) -> None: @@ -318,6 +350,7 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): """Adapt request-mode scheduler output to worker execute_model RPCs. Returns a BatchRunnerOutput with one RunnerOutput per scheduled request. + In async mode the result may carry async_output_id instead of the final output. """ from vllm_omni.diffusion.worker.utils import BatchRunnerOutput, RunnerOutput @@ -333,16 +366,27 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): unique_reply_rank=0, exec_all_ranks=True, ) - if not isinstance(result, DiffusionOutput): - raise RuntimeError(f"Unexpected response type: {type(result)!r}") - runner_outputs.append( - RunnerOutput( - request_id=new_req.request_id, - step_index=None, - finished=True, - result=result, + if isinstance(result, AsyncDiffusionOutput) and result.kind == AsyncOutputKind.COMPUTE_DONE: + runner_outputs.append( + RunnerOutput( + request_id=new_req.request_id, + step_index=None, + finished=True, + result=None, + async_output_id=result.async_output_id, + ) ) - ) + elif isinstance(result, DiffusionOutput): + runner_outputs.append( + RunnerOutput( + request_id=new_req.request_id, + step_index=None, + finished=True, + result=result, + ) + ) + else: + raise RuntimeError(f"Unexpected response type: {type(result)!r}") except Exception as exc: runner_outputs.append( RunnerOutput( @@ -361,8 +405,12 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): A scheduler wave with one request is the conservative serial case and uses the single-request worker RPC. Waves with multiple requests use the fused request-batch RPC and require pipeline request-batch support. + + When async output is enabled, per-request async_output_ids are + propagated through RunnerOutput so the engine waits in + step_streaming() instead of blocking the busy loop here. """ - from vllm_omni.diffusion.worker.utils import BatchRunnerOutput + from vllm_omni.diffusion.worker.utils import BatchRunnerOutput, RunnerOutput self._ensure_open() if len(scheduler_output.scheduled_new_reqs) <= 1: @@ -374,6 +422,27 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): unique_reply_rank=0, exec_all_ranks=True, ) + if isinstance(result, AsyncDiffusionOutput) and result.kind == AsyncOutputKind.COMPUTE_DONE: + # Propagate async_output_id to per-request RunnerOutputs so the + # engine waits in step_streaming() instead of blocking here. + batch_id = result.async_output_id + per_req_map: dict[str, str] = {} + runner_outputs: list[RunnerOutput] = [] + for new_req in scheduler_output.scheduled_new_reqs: + per_req_id = f"{batch_id}/{new_req.request_id}" + per_req_map[per_req_id] = new_req.request_id + runner_outputs.append( + RunnerOutput( + request_id=new_req.request_id, + step_index=None, + finished=True, + result=None, + async_output_id=per_req_id, + ) + ) + with self._futures_lock: + self._batch_split_map[batch_id] = per_req_map + return BatchRunnerOutput.from_list(runner_outputs) if not isinstance(result, BatchRunnerOutput): raise RuntimeError(f"Unexpected response type for execute_batch: {type(result)!r}") return result @@ -408,11 +477,6 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): deadline = None if timeout is None else time.monotonic() + timeout kwargs = kwargs or {} - # Prepare RPC request message. When unique_reply_rank is None, all - # workers must execute the RPC but only rank 0 can reply (it's the - # only one with a result_mq). Collect detailed rank statuses only for - # this control-plane all-rank path; forward-path exec_all_ranks RPCs - # avoid the per-step host object gather. execute_all_ranks = unique_reply_rank is None or exec_all_ranks collect_rank_status = unique_reply_rank is None rpc_request = { @@ -425,11 +489,32 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): "collect_rank_status": collect_rank_status, } + # ── Path 1: async execute_model / execute_model_batch ── + # Request-mode methods use the compute_done/output_ready split + # so the D2H copy runs on a side stream in the worker background + # thread while the default stream is free for the next forward. + # pump routes the result back to a Future via rpc_id. + if not self.od_config.step_execution and method in ("execute_model", "execute_model_batch"): + rpc_id = self._next_rpc_id() + rpc_request["rpc_id"] = rpc_id + fut: concurrent.futures.Future = concurrent.futures.Future() + with self._futures_lock: + self._rpc_futures[rpc_id] = fut + self._broadcast_mq.enqueue(rpc_request) # pyright: ignore[reportOptionalMemberAccess] MQ is not None before shutdown + try: + return fut.result(timeout=timeout) + except concurrent.futures.TimeoutError: + with self._futures_lock: + self._rpc_futures.pop(rpc_id, None) + raise TimeoutError(f"RPC call to {method} timed out.") + + # ── Path 2: step-execution or other non-execute_model RPCs ── + # _dequeue_one_with_failure_polling reads from _sync_result_buffer + # when pump is active (request-mode), or result_mq directly otherwise. try: # Broadcast RPC request to all workers via unified message queue self._broadcast_mq.enqueue(rpc_request) # pyright: ignore[reportOptionalMemberAccess] MQ is not None before shutdown - # Only rank 0 has a result_mq, so we always expect exactly 1 response num_responses = 1 responses = [] @@ -450,6 +535,123 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): logger.error(f"RPC call failed: {e}") raise + # ------------------------------------------------------------------ + # Async output: result pump + # ------------------------------------------------------------------ + + def _start_result_pump(self) -> None: + self._pump_running = True + self._pump_stop.clear() + self._result_pump_thread = threading.Thread(target=self._result_pump, daemon=True, name="DiffusionResultPump") + self._result_pump_thread.start() + logger.info("Async result pump started") + + def _result_pump(self) -> None: + """Sole reader of result_mq when async output is enabled. + + Dispatches AsyncDiffusionOutput messages to the appropriate future: + * RPC_RESULT / COMPUTE_DONE → _rpc_futures[rpc_id] + * OUTPUT_READY → _output_futures[async_output_id] + """ + while not self._pump_stop.is_set(): + try: + msg = self._result_mq.dequeue(timeout=1.0) + except TimeoutError: + if self._is_failed: + break + continue + except Exception: + logger.exception("Result pump dequeue failed") + if self._is_failed: + break + continue + + if not isinstance(msg, AsyncDiffusionOutput): + # Non-async message: place into the sync buffer for + # collective_rpc() to consume via Path 2. + self._sync_result_buffer.put(msg) + continue + + if msg.kind in (AsyncOutputKind.RPC_RESULT, AsyncOutputKind.COMPUTE_DONE): + with self._futures_lock: + fut = self._rpc_futures.pop(msg.rpc_id, None) if msg.rpc_id else None + if fut is not None and not fut.done(): + if msg.error: + fut.set_exception(RuntimeError(msg.error)) + else: + fut.set_result(msg) + elif msg.kind == AsyncOutputKind.OUTPUT_READY: + batch_id = msg.async_output_id + with self._futures_lock: + per_req_map = self._batch_split_map.pop(batch_id, None) if batch_id else None + if per_req_map is not None: + # Batch result: split into per-request DiffusionOutputs. + try: + unpack_diffusion_output_shm(msg.output) + except Exception: + logger.exception("SHM unpack failed for batch %s", batch_id) + batch_output = msg.output + for per_req_id, req_id in per_req_map.items(): + req_output = batch_output.get_request_output(req_id) + per_req_result: DiffusionOutput + if req_output is not None and req_output.result is not None: + per_req_result = req_output.result + elif msg.error: + per_req_result = DiffusionOutput(error=msg.error) + else: + per_req_result = DiffusionOutput(error="No output result for batch request") + fut: concurrent.futures.Future = concurrent.futures.Future() + fut.set_result(per_req_result) + with self._futures_lock: + pending = self._output_futures.pop(per_req_id, None) + if pending is not None and not pending.done(): + pending.set_result(per_req_result) + else: + self._completed_outputs[per_req_id] = fut + else: + with self._futures_lock: + fut = self._output_futures.pop(batch_id, None) if batch_id else None + if fut is not None and not fut.done(): + if msg.error: + fut.set_exception(RuntimeError(msg.error)) + else: + try: + unpack_diffusion_output_shm(msg.output) + except Exception as e: + logger.exception("SHM unpack failed in result pump") + fut.set_exception(e) + continue + fut.set_result(msg.output) + elif batch_id: + fut = concurrent.futures.Future() + if msg.error: + fut.set_exception(RuntimeError(msg.error)) + else: + try: + unpack_diffusion_output_shm(msg.output) + except Exception as e: + logger.exception("SHM unpack failed in result pump (cached)") + fut.set_exception(e) + else: + fut.set_result(msg.output) + with self._futures_lock: + self._completed_outputs[batch_id] = fut + + def _next_rpc_id(self) -> str: + with self._rpc_id_lock: + self._rpc_id_counter += 1 + return str(self._rpc_id_counter) + + def wait_output_ready(self, async_output_id: str) -> concurrent.futures.Future[DiffusionOutput]: + """Return a Future that resolves when the async output is ready.""" + with self._futures_lock: + cached = self._completed_outputs.pop(async_output_id, None) + if cached is not None: + return cached + fut: concurrent.futures.Future = concurrent.futures.Future() + self._output_futures[async_output_id] = fut + return fut + def check_health(self) -> None: if self._is_failed: raise EngineDeadError() @@ -461,10 +663,21 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): def shutdown(self) -> None: self._closed = True + self._pump_stop.set() try: self._finalizer() finally: self._broadcast_mq = None self._result_mq = None + with self._futures_lock: + for fut in self._rpc_futures.values(): + if not fut.done(): + fut.set_exception(RuntimeError("Executor shut down")) + for fut in self._output_futures.values(): + if not fut.done(): + fut.set_exception(RuntimeError("Executor shut down")) + self._rpc_futures.clear() + self._output_futures.clear() + self._batch_split_map.clear() self._shutdown_cleaner = None self._processes = [] diff --git a/vllm_omni/diffusion/ipc.py b/vllm_omni/diffusion/ipc.py index cd83e718c..ca6bc858d 100644 --- a/vllm_omni/diffusion/ipc.py +++ b/vllm_omni/diffusion/ipc.py @@ -21,23 +21,43 @@ _SHM_TENSOR_THRESHOLD = 1_000_000 # 1 MB DIFFUSION_RPC_RESULT_ENVELOPE = "diffusion_rpc_result" -def _tensor_to_shm(tensor: torch.Tensor) -> dict[str, Any]: +def _tensor_to_shm( + tensor: torch.Tensor, + d2h_stream: torch.Stream | None = None, +) -> dict[str, Any]: """Copy a tensor into POSIX shared memory and return a metadata handle. The shared memory segment remains alive after this call (the local fd is closed, but the segment persists until ``_tensor_from_shm`` unlinks it). + + If *d2h_stream* is provided, the D2H copy uses ``copy_()`` with + ``pin_memory=True`` on that stream instead of the synchronous ``.cpu()`` + path. The caller must synchronize *d2h_stream* after all tensors are + packed. """ from multiprocessing import shared_memory import numpy as np - tensor = tensor.detach().cpu().contiguous() original_dtype = tensor.dtype - # NumPy does not support bfloat16; promote to float32 for the SHM - # transfer and record the original dtype so _tensor_from_shm can - # convert back. The round-trip is lossless for bfloat16 values. - if original_dtype == torch.bfloat16: - tensor = tensor.to(torch.float32) + if d2h_stream is not None: + # Non-blocking D2H: copy on side stream to pinned CPU memory. + old_stream = torch.accelerator.current_stream() + torch.accelerator.set_stream(d2h_stream) + try: + t = tensor.detach() + if original_dtype == torch.bfloat16: + t = t.to(torch.float32) + cpu = torch.empty(t.shape, dtype=t.dtype, pin_memory=True) + cpu.copy_(t, non_blocking=True) + finally: + torch.accelerator.set_stream(old_stream) + d2h_stream.synchronize() + tensor = cpu + else: + tensor = tensor.detach().cpu().contiguous() + if original_dtype == torch.bfloat16: + tensor = tensor.to(torch.float32) arr = tensor.numpy() nbytes = arr.nbytes shm = shared_memory.SharedMemory(create=True, size=nbytes) @@ -79,14 +99,20 @@ def _tensor_from_shm(handle: dict[str, Any]) -> torch.Tensor: return tensor -def _pack_tensor_if_large(val: torch.Tensor) -> torch.Tensor | dict: +def _pack_tensor_if_large( + val: torch.Tensor, + d2h_stream: torch.Stream | None = None, +) -> torch.Tensor | dict: """Replace a tensor with an SHM handle if it exceeds the threshold.""" if val.nelement() * val.element_size() > _SHM_TENSOR_THRESHOLD: - return _tensor_to_shm(val) + return _tensor_to_shm(val, d2h_stream=d2h_stream) return val -def _pack_value_if_large(val: object) -> object: +def _pack_value_if_large( + val: object, + d2h_stream: torch.Stream | None = None, +) -> object: """Recursively replace large tensors with SHM handles. Walks the container shapes pipelines return as ``DiffusionOutput.output``: @@ -96,13 +122,13 @@ def _pack_value_if_large(val: object) -> object: the two in sync. """ if isinstance(val, torch.Tensor): - return _pack_tensor_if_large(val) + return _pack_tensor_if_large(val, d2h_stream=d2h_stream) if isinstance(val, dict): - return {key: _pack_value_if_large(value) for key, value in val.items()} + return {key: _pack_value_if_large(value, d2h_stream=d2h_stream) for key, value in val.items()} if isinstance(val, list): - return [_pack_value_if_large(item) for item in val] + return [_pack_value_if_large(item, d2h_stream=d2h_stream) for item in val] if isinstance(val, tuple): - return tuple(_pack_value_if_large(item) for item in val) + return tuple(_pack_value_if_large(item, d2h_stream=d2h_stream) for item in val) return val @@ -119,15 +145,18 @@ def _unpack_if_shm_handle(val: object) -> object: return val -def _pack_diffusion_fields(output: DiffusionOutput) -> DiffusionOutput: +def _pack_diffusion_fields( + output: DiffusionOutput, + d2h_stream: torch.Stream | None = None, +) -> DiffusionOutput: if output.output is not None: - output.output = _pack_value_if_large(output.output) + output.output = _pack_value_if_large(output.output, d2h_stream=d2h_stream) if output.trajectory_latents is not None and isinstance(output.trajectory_latents, torch.Tensor): - output.trajectory_latents = _pack_tensor_if_large(output.trajectory_latents) + output.trajectory_latents = _pack_tensor_if_large(output.trajectory_latents, d2h_stream=d2h_stream) if output.trajectory_timesteps is not None and isinstance(output.trajectory_timesteps, torch.Tensor): - output.trajectory_timesteps = _pack_tensor_if_large(output.trajectory_timesteps) + output.trajectory_timesteps = _pack_tensor_if_large(output.trajectory_timesteps, d2h_stream=d2h_stream) if output.trajectory_log_probs is not None and isinstance(output.trajectory_log_probs, torch.Tensor): - output.trajectory_log_probs = _pack_tensor_if_large(output.trajectory_log_probs) + output.trajectory_log_probs = _pack_tensor_if_large(output.trajectory_log_probs, d2h_stream=d2h_stream) return output @@ -135,31 +164,37 @@ def _is_rpc_result_envelope(output: object) -> bool: return isinstance(output, dict) and output.get("type") == DIFFUSION_RPC_RESULT_ENVELOPE -def pack_diffusion_output_shm(output: object) -> object: +def pack_diffusion_output_shm( + output: object, + d2h_stream: torch.Stream | None = None, +) -> object: """Replace large tensors in diffusion worker outputs with SHM handles. Supports a bare ``DiffusionOutput``, a wrapper object carrying one in ``.result`` (for example ``RunnerOutput``), an RPC result envelope carrying the diffusion output in ``["result"]``, or a batch wrapper carrying ``RunnerOutput`` objects in ``.runner_outputs``. + + If *d2h_stream* is provided, D2H copies use that stream (non-blocking on + the default stream). The caller must synchronize *d2h_stream* afterward. """ if isinstance(output, DiffusionOutput): - return _pack_diffusion_fields(output) + return _pack_diffusion_fields(output, d2h_stream=d2h_stream) if _is_rpc_result_envelope(output): result = output.get("result") if isinstance(result, DiffusionOutput): - output["result"] = _pack_diffusion_fields(result) + output["result"] = _pack_diffusion_fields(result, d2h_stream=d2h_stream) return output result = getattr(output, "result", None) if isinstance(result, DiffusionOutput): - output.result = _pack_diffusion_fields(result) + output.result = _pack_diffusion_fields(result, d2h_stream=d2h_stream) runner_outputs = getattr(output, "runner_outputs", None) if isinstance(runner_outputs, list): for runner_output in runner_outputs: - pack_diffusion_output_shm(runner_output) + pack_diffusion_output_shm(runner_output, d2h_stream=d2h_stream) return output diff --git a/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py b/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py index f749d73d3..877fd1c3f 100644 --- a/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py +++ b/vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py @@ -3233,8 +3233,7 @@ class HunyuanImage3Text2ImagePipeline(DiffusionPipeline): assert image.shape[2] == 1, "image should have shape [B, C, T, H, W] and T should be 1" image = image.squeeze(2) - do_denormalize = [True] * image.shape[0] - image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + # Denormalize + PIL moved to engine post_process_func for overlap with next request. if not return_dict: return (image,) diff --git a/vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py b/vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py index 095a420c8..36e69302e 100644 --- a/vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py +++ b/vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py @@ -71,6 +71,8 @@ _STEP_COT_TEXT_LIST = "hunyuan_cot_text_list" _STEP_AR_KV = "hunyuan_ar_kv" _STEP_PROMPT_KV = "hunyuan_prompt_kv" +_HUNYUAN_DEFAULT_OUTPUT_TYPE = "pil" + def default(val, d): return val if val is not None else d @@ -333,6 +335,41 @@ def get_hunyuan_image_3_pre_process_func(od_config: OmniDiffusionConfig): return pre_process_func +def get_hunyuan_image3_post_process_func(od_config: OmniDiffusionConfig): + """GPU tensor → PIL, runs outside pipeline_forward to overlap with next request.""" + from diffusers.image_processor import VaeImageProcessor + + image_processor = VaeImageProcessor() + + def post_process_func(images: torch.Tensor | dict): + # Handle dict envelope format (upstream): {"payload": {"image": ...}, "metadata": ...} + if isinstance(images, dict) and "payload" in images: + tensor = images["payload"].get("image") + if tensor is None: + return images + if tensor.dim() == 3: + tensor = tensor.unsqueeze(0) + do_denormalize = [True] * tensor.shape[0] + images["payload"]["image"] = image_processor.postprocess( + tensor, + output_type=_HUNYUAN_DEFAULT_OUTPUT_TYPE, + do_denormalize=do_denormalize, + ) + return images + + # Legacy raw tensor format + if images.dim() == 3: + images = images.unsqueeze(0) + do_denormalize = [True] * images.shape[0] + return image_processor.postprocess( + images, + output_type=_HUNYUAN_DEFAULT_OUTPUT_TYPE, + do_denormalize=do_denormalize, + ) + + return post_process_func + + class HunyuanImage3Pipeline( HunyuanImage3PreTrainedModel, GenerationMixin, @@ -2224,11 +2261,7 @@ class HunyuanImage3Pipeline( if hasattr(self.vae, "ffactor_temporal"): assert image.shape[2] == 1, "image should have shape [B, C, T, H, W] and T should be 1" image = image.squeeze(2) - image = self.pipeline.image_processor.postprocess( - image, - output_type=output_type, - do_denormalize=[True] * image.shape[0], - ) + # Postprocess deferred to engine post_process_func for overlap with next request. cot_text_list = state.extra.get(_STEP_COT_TEXT_LIST) or [] metadata = {} diff --git a/vllm_omni/diffusion/registry.py b/vllm_omni/diffusion/registry.py index aaedc7b00..1fbddcb96 100644 --- a/vllm_omni/diffusion/registry.py +++ b/vllm_omni/diffusion/registry.py @@ -530,6 +530,7 @@ _DIFFUSION_POST_PROCESS_FUNCS = { "Flux2Pipeline": "get_flux2_post_process_func", "HunyuanVideo15Pipeline": "get_hunyuan_video_15_post_process_func", "HunyuanVideo15ImageToVideoPipeline": "get_hunyuan_video_15_i2v_post_process_func", + "HunyuanImage3Pipeline": "get_hunyuan_image3_post_process_func", "LingBotVideoPipeline": "get_lingbot_video_post_process_func", "MagiHumanPipeline": "get_magi_human_post_process_func", "OmniVoicePipeline": "get_omnivoice_post_process_func", @@ -540,6 +541,7 @@ _DIFFUSION_POST_PROCESS_FUNCS = { "HiDreamImagePipeline": "get_hidream_image_post_process_func", "StableDiffusionXLPipeline": "get_sdxl_image_post_process_func", "Krea2Pipeline": "get_krea2_post_process_func", + "HunyuanImage3ForCausalMM": "get_hunyuan_image3_post_process_func", } _DIFFUSION_IR_OP_PRIORITY_FUNCS = { diff --git a/vllm_omni/diffusion/sched/request_scheduler.py b/vllm_omni/diffusion/sched/request_scheduler.py index 9e7442d3a..ec6b2aa8e 100644 --- a/vllm_omni/diffusion/sched/request_scheduler.py +++ b/vllm_omni/diffusion/sched/request_scheduler.py @@ -50,8 +50,14 @@ class RequestScheduler(BaseScheduler): req_output = output.get_request_output(request_id) result = req_output.result if req_output is not None else None if result is None: - terminal_statuses[request_id] = DiffusionRequestStatus.FINISHED_ERROR - terminal_errors[request_id] = "No output result" + # Async mode: result=None with async_output_id means compute done, + # final output will arrive later via wait_output_ready. + if req_output is not None and req_output.async_output_id is not None: + terminal_statuses[request_id] = DiffusionRequestStatus.FINISHED_COMPLETED + terminal_errors[request_id] = None + else: + terminal_statuses[request_id] = DiffusionRequestStatus.FINISHED_ERROR + terminal_errors[request_id] = "No output result" elif result.aborted: terminal_statuses[request_id] = DiffusionRequestStatus.FINISHED_ABORTED terminal_errors[request_id] = None diff --git a/vllm_omni/diffusion/worker/diffusion_worker.py b/vllm_omni/diffusion/worker/diffusion_worker.py index b7450038a..58c8fd1eb 100644 --- a/vllm_omni/diffusion/worker/diffusion_worker.py +++ b/vllm_omni/diffusion/worker/diffusion_worker.py @@ -11,8 +11,11 @@ to DiffusionModelRunner. import gc import multiprocessing as mp import os +import queue import signal +import threading import traceback +import uuid from collections.abc import Iterator from contextlib import AbstractContextManager, contextmanager, nullcontext from dataclasses import dataclass @@ -30,6 +33,8 @@ from vllm.utils.mem_utils import GiB_bytes from vllm.v1.worker.workspace import init_workspace_manager from vllm_omni.diffusion.data import ( + AsyncDiffusionOutput, + AsyncOutputKind, DiffusionOutput, OmniACK, OmniDiffusionConfig, @@ -794,6 +799,21 @@ class WorkerProc: self.worker = self._create_worker(gpu_id, od_config, worker_extension_cls, custom_pipeline_args) self._running = True + self._async_output_queue: queue.Queue | None = None + self._async_output_thread: threading.Thread | None = None + if not self.od_config.step_execution: + self._async_output_queue = queue.Queue() + self._async_output_thread = threading.Thread( + target=self._async_output_loop, + daemon=True, + name="DiffusionAsyncOutput", + ) + self._async_output_thread.start() + + @staticmethod + def _generate_async_output_id() -> str: + return uuid.uuid4().hex + def _create_worker( self, gpu_id: int, @@ -813,18 +833,72 @@ class WorkerProc: ) return wrapper - def _return_result(self, output: Any) -> None: + def _return_result(self, output: Any, rpc_id: str | None = None) -> None: """Reply to client, only on rank 0.""" - if self.result_mq is not None: - if isinstance(output, OmniACK): - self.result_mq.enqueue(output) - return - try: - pack_diffusion_output_shm(output) - except Exception as e: - if hasattr(output, "output"): - logger.warning("SHM pack failed for model output: %s", e) + if self.result_mq is None: + return + if isinstance(output, OmniACK): self.result_mq.enqueue(output) + return + + # Async path: enqueue compute_done immediately, bg thread does D2H+SHM. + if not self.od_config.step_execution and isinstance(output, (DiffusionOutput, BatchRunnerOutput)): + async_output_id = WorkerProc._generate_async_output_id() + gpu_event = current_omni_platform.record_device_event() + self._async_output_queue.put((output, async_output_id, gpu_event)) + msg = AsyncDiffusionOutput( + kind=AsyncOutputKind.COMPUTE_DONE, + rpc_id=rpc_id, + async_output_id=async_output_id, + ) + self.result_mq.enqueue(msg) + return + + # Sync path (original, or async fallback). + try: + pack_diffusion_output_shm(output) + except Exception as e: + if hasattr(output, "output"): + logger.warning("SHM pack failed for model output: %s", e) + self.result_mq.enqueue(output) + + def _async_output_loop(self): + """Background thread: D2H + SHM packing for async diffusion output. + + Uses a side stream so the D2H transfer does not block the default + stream where the next forward runs. + """ + device = torch.device(torch.accelerator.current_accelerator().type, self.gpu_id) + d2h_stream = torch.Stream(device=device) + while self._running: + output, async_output_id, gpu_event = self._async_output_queue.get() + try: + # Cross-stream ordering: wait for default stream to finish + # writing the output tensors before the side stream reads. + if gpu_event is not None: + d2h_stream.wait_event(gpu_event) + pack_diffusion_output_shm(output, d2h_stream=d2h_stream) + d2h_stream.synchronize() + + self.result_mq.enqueue( + AsyncDiffusionOutput( + kind=AsyncOutputKind.OUTPUT_READY, + async_output_id=async_output_id, + output=output, + ) + ) + except Exception: + logger.exception( + "Async output packing failed for id '%s'; sending error", + async_output_id, + ) + self.result_mq.enqueue( + AsyncDiffusionOutput( + kind=AsyncOutputKind.OUTPUT_READY, + async_output_id=async_output_id, + error="Background D2H/SHM packing failed", + ) + ) def _gather_rpc_rank_statuses(self, status: dict[str, Any]) -> list[dict[str, Any]]: if not torch.distributed.is_initialized(): @@ -936,13 +1010,25 @@ class WorkerProc: # Route message based on type elif isinstance(msg, dict) and msg.get("type") == "rpc": try: + rpc_id = msg.get("rpc_id") result, should_reply = self._execute_rpc(msg) if should_reply: - self._return_result(result) + self._return_result(result, rpc_id=rpc_id) except Exception as e: logger.error(f"Error processing RPC: {e}", exc_info=True) if self.result_mq is not None: - self._return_result({"status": "error", "error": str(e)}) + if rpc_id is not None: + # Async RPC: must complete the executor''s pending + # future so collective_rpc() doesn''t hang. + self.result_mq.enqueue( + AsyncDiffusionOutput( + kind=AsyncOutputKind.RPC_RESULT, + rpc_id=rpc_id, + error=str(e), + ) + ) + else: + self._return_result({"status": "error", "error": str(e)}) elif isinstance(msg, dict) and msg.get("type") == "shutdown": logger.info("Worker %s: Received shutdown message", self.gpu_id) diff --git a/vllm_omni/diffusion/worker/utils.py b/vllm_omni/diffusion/worker/utils.py index b49388fee..40ed550d0 100644 --- a/vllm_omni/diffusion/worker/utils.py +++ b/vllm_omni/diffusion/worker/utils.py @@ -183,6 +183,7 @@ class RunnerOutput(BaseRunnerOutput): step_index: int | None = None finished: bool = False result: DiffusionOutput | None = None + async_output_id: str | None = None def get_request_output(self, request_id: str) -> RunnerOutput | None: return self if self.request_id == request_id else None diff --git a/vllm_omni/platforms/cuda/platform.py b/vllm_omni/platforms/cuda/platform.py index 5eb7406a5..642d8ec02 100644 --- a/vllm_omni/platforms/cuda/platform.py +++ b/vllm_omni/platforms/cuda/platform.py @@ -273,6 +273,17 @@ class CudaOmniPlatform(OmniPlatform, CudaPlatformBase): def synchronize(cls) -> None: torch.accelerator.synchronize() + @classmethod + def record_device_event(cls) -> torch.Event | None: + """Record a device event on the default stream to mark tensor readiness.""" + try: + event = torch.Event() + event.record() + return event + except Exception: + logger.warning("Failed to record device event for cross-stream sync") + return None + @classmethod def get_free_memory(cls, device: torch.device | None = None) -> int: free, _ = torch.cuda.mem_get_info(device) diff --git a/vllm_omni/platforms/interface.py b/vllm_omni/platforms/interface.py index 9969849d9..99dafa2f2 100644 --- a/vllm_omni/platforms/interface.py +++ b/vllm_omni/platforms/interface.py @@ -190,6 +190,24 @@ class OmniPlatform(Platform): def synchronize(cls) -> None: raise NotImplementedError + # ── Async diffusion output: cross-stream sync ── + + @classmethod + def record_device_event(cls): + """Record a device event on the default stream to mark tensor readiness. + + On platforms where distributed communication (e.g. HCCL) may use + internal streams not visible to the default stream, this method + should synchronize the default stream before recording the event + to ensure the event captures all completed work including + cross-device communication results. + + Returns ``None`` by default so that platforms without a native + implementation (ROCm, XPU, MUSA) fall through to a safe no-op. + Override in platform subclasses to provide real event support. + """ + return None + @classmethod def get_free_memory(cls, device: torch.device | None = None) -> int: raise NotImplementedError diff --git a/vllm_omni/platforms/musa/platform.py b/vllm_omni/platforms/musa/platform.py index 818338da4..c71b5a70b 100644 --- a/vllm_omni/platforms/musa/platform.py +++ b/vllm_omni/platforms/musa/platform.py @@ -152,6 +152,16 @@ class MUSAOmniPlatform(OmniPlatform, MUSAPlatformBase): """Synchronize all MUSA operations.""" torch.musa.synchronize() + @classmethod + def record_device_event(cls) -> torch.Event | None: + try: + event = torch.musa.Event() + event.record() + return event + except Exception: + logger.warning("Failed to record MUSA device event for cross-stream sync") + return None + @classmethod def get_free_memory(cls, device: torch.device | None = None) -> int: free, _ = torch.musa.mem_get_info(device) diff --git a/vllm_omni/platforms/npu/platform.py b/vllm_omni/platforms/npu/platform.py index 240f9d576..bb0e1b436 100644 --- a/vllm_omni/platforms/npu/platform.py +++ b/vllm_omni/platforms/npu/platform.py @@ -175,6 +175,25 @@ class NPUOmniPlatform(OmniPlatform, NPUPlatform): def synchronize(cls) -> None: torch.npu.synchronize() + @classmethod + def record_device_event(cls) -> torch.Event | None: + """Record a NPU event on the default stream to mark tensor readiness. + + On NPU/Ascend with HCCL, distributed communication may use internal + streams not visible to the default stream. Synchronize the default + stream first so that HCCL results are written back before we record + the event, ensuring d2h_stream.wait_event() captures the complete + output data. + """ + try: + torch.npu.current_stream().synchronize() + event = torch.npu.Event() + event.record() + return event + except Exception: + logger.warning("Failed to record NPU event for cross-stream sync") + return None + @classmethod def get_free_memory(cls, device: torch.device | None = None) -> int: free, _ = torch.npu.mem_get_info(device) diff --git a/vllm_omni/platforms/rocm/platform.py b/vllm_omni/platforms/rocm/platform.py index f44bb86fa..cf72baccb 100644 --- a/vllm_omni/platforms/rocm/platform.py +++ b/vllm_omni/platforms/rocm/platform.py @@ -145,6 +145,16 @@ class RocmOmniPlatform(OmniPlatform, RocmPlatform): def synchronize(cls) -> None: torch.accelerator.synchronize() + @classmethod + def record_device_event(cls) -> torch.Event | None: + try: + event = torch.Event() + event.record() + return event + except Exception: + logger.warning("Failed to record device event for cross-stream sync") + return None + @classmethod def get_free_memory(cls, device: torch.device | None = None) -> int: free, _ = torch.cuda.mem_get_info(device) diff --git a/vllm_omni/platforms/xpu/platform.py b/vllm_omni/platforms/xpu/platform.py index 26d3dccde..aa26cd82c 100644 --- a/vllm_omni/platforms/xpu/platform.py +++ b/vllm_omni/platforms/xpu/platform.py @@ -99,6 +99,16 @@ class XPUOmniPlatform(OmniPlatform, XPUPlatform): def synchronize(cls) -> None: torch.xpu.synchronize() + @classmethod + def record_device_event(cls) -> torch.Event | None: + try: + event = torch.xpu.Event() + event.record() + return event + except Exception: + logger.warning("Failed to record XPU device event for cross-stream sync") + return None + @classmethod def get_free_memory(cls, device: torch.device | None = None) -> int: free, _ = torch.xpu.mem_get_info(device) From 1b718f7b1529bf7b59450de2dba0f820ef676dc8 Mon Sep 17 00:00:00 2001 From: Mu GuanLin <1203789601@qq.com> Date: Thu, 30 Jul 2026 01:40:40 +0800 Subject: [PATCH 26/37] [Feat] [CI] [bugfix] Generalize LTX CFG parallelism to the complete guidance plan (#5547) Signed-off-by: mglyn <1203789601@qq.com> --- recipes/LTX/LTX-2.md | 38 ++- tests/config/test_omni_config.py | 8 +- .../models/ltx2/test_ltx2_conditioning.py | 2 +- .../models/ltx2/test_ltx2_parallel.py | 182 +++++++++++++- tests/entrypoints/test_serve.py | 10 + vllm_omni/config/omni_config.py | 2 +- vllm_omni/diffusion/data.py | 5 +- .../models/ltx2/ltx2_conditioning.py | 8 +- .../diffusion/models/ltx2/ltx2_denoise.py | 12 +- .../diffusion/models/ltx2/ltx2_guidance.py | 235 ++++++++++++------ .../diffusion/models/ltx2/ltx2_runtime.py | 53 ++-- vllm_omni/entrypoints/cli/serve.py | 3 +- 12 files changed, 419 insertions(+), 139 deletions(-) diff --git a/recipes/LTX/LTX-2.md b/recipes/LTX/LTX-2.md index ca3201dc1..fa9ec6755 100644 --- a/recipes/LTX/LTX-2.md +++ b/recipes/LTX/LTX-2.md @@ -165,6 +165,40 @@ Pass these fields in online `extra_params` or offline `--extra-body`: --extra-body '{"video_cfg_scale":3.0,"audio_cfg_scale":7.0}' ``` +### Guidance Parallelism + +For LTX, `cfg_parallel_size` is the number of ranks used to execute the +complete guidance plan in parallel. Despite the legacy `cfg` name, it covers +text CFG, STG, and cross-modality guidance passes; guidance rescaling is +applied after all pass predictions are gathered. + +The default one-stage recipe has four Transformer passes per denoise step: +`cond`, `uncond`, `ptb` (STG), and `mod` (cross-modality). The useful balanced +configurations are therefore: + +| `--cfg-parallel-size` | Passes per rank | Guidance-slot utilization | Notes | +|---:|---:|---:|---| +| `1` | 4 | 100% | Single-rank fused guidance batch | +| `2` | 2 | 100% | Recommended two-rank configuration | +| `4` | 1 | 100% | One guidance pass per rank | + +Other positive sizes are accepted. When the pass count is not divisible by +`cfg_parallel_size`, ranks are padded to an equal number of execution slots +and LTX emits a warning with the expected utilization. For example, four +passes on three ranks use six slots and have 66.7% guidance-slot utilization. + +Start an LTX-2.3 server with two-way guidance parallelism: + +```bash +vllm serve diffusers/LTX-2.3-Diffusers --omni \ + --cfg-parallel-size 2 --stage-init-timeout 600 +``` + +The total device count is the product of `cfg_parallel_size` and the other +configured parallel dimensions. Distilled pipelines use positive-only +guidance, so `cfg_parallel_size > 1` does not provide guidance speedup for +them. + ### Python API Normal per-request values belong in `OmniDiffusionSamplingParams`. Pass the @@ -273,8 +307,8 @@ bundled offline CLI do not currently expose `sigmas`. - Fused one-stage requests must resolve to identical LTX guidance and sigma schedules. Keep concurrent requests guidance-homogeneous or use `--max-num-seqs 1`. -- `--cfg-parallel-size 2` supports only the CFG-only plan: disable STG and - modality guidance and set both rescale fields to `0.0`. +- `--cfg-parallel-size` shards the complete LTX guidance plan, including STG, + modality guidance, and rescale-compatible prediction gathering. ## Operational Notes diff --git a/tests/config/test_omni_config.py b/tests/config/test_omni_config.py index febaf4b0b..d477e05c1 100644 --- a/tests/config/test_omni_config.py +++ b/tests/config/test_omni_config.py @@ -577,9 +577,11 @@ def test_from_pipeline_config_derives_sequence_parallel_size_from_allgather_degr assert stage.parallel_config.world_size == 2 -def test_diffusion_parallel_config_rejects_cfg_parallel_size_outside_current_bound(): - with pytest.raises(ValidationError): - OmniStageDiffusionParallelConfig(cfg_parallel_size=4) +def test_diffusion_parallel_config_accepts_four_way_guidance_parallelism(): + cfg = OmniStageDiffusionParallelConfig(cfg_parallel_size=4) + + assert cfg.cfg_parallel_size == 4 + assert cfg.world_size == 4 def test_diffusion_parallel_config_rejects_allgather_with_ulysses_or_ring(): diff --git a/tests/diffusion/models/ltx2/test_ltx2_conditioning.py b/tests/diffusion/models/ltx2/test_ltx2_conditioning.py index 6b4ec70ce..e309dbf47 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_conditioning.py +++ b/tests/diffusion/models/ltx2/test_ltx2_conditioning.py @@ -110,7 +110,7 @@ class TestLTXImageToVideoForwardStages: kwargs = pipe._denoise_timestep_kwargs( ts, - SimpleNamespace(cfg_parallel_ready=False), + SimpleNamespace(guidance_parallel_ready=False), denoise_ctx, video_token_count=2, audio_token_count=1, diff --git a/tests/diffusion/models/ltx2/test_ltx2_parallel.py b/tests/diffusion/models/ltx2/test_ltx2_parallel.py index 92077a229..8d8c22e79 100644 --- a/tests/diffusion/models/ltx2/test_ltx2_parallel.py +++ b/tests/diffusion/models/ltx2/test_ltx2_parallel.py @@ -3,6 +3,7 @@ """Unit tests for LTX guidance and forward-parallel behavior.""" +from contextlib import nullcontext from types import SimpleNamespace import pytest @@ -50,7 +51,8 @@ class TestCFGParallelHelpers: assert torch.allclose(video_combined, expected_video) assert torch.allclose(audio_combined, expected_audio) - def test_cfg_parallel_rejects_non_cfg_guidance(self): + @pytest.mark.parametrize("guidance_world_size", [1, 2, 3, 4]) + def test_guidance_parallel_accepts_official_guidance(self, guidance_world_size): from vllm_omni.diffusion.models.ltx2.ltx2_guidance import ( LTX_GUIDANCE_EXECUTOR, LTXGuidancePlan, @@ -59,26 +61,180 @@ class TestCFGParallelHelpers: plan = LTXGuidancePlan.build(LTX23_ONE_STAGE_RECIPE.request_guidance) - with pytest.raises(ValueError, match="CFG-only guidance"): - LTX_GUIDANCE_EXECUTOR.validate_cfg_world_size(plan, 2) + LTX_GUIDANCE_EXECUTOR.validate_guidance_world_size(plan, guidance_world_size) - def test_cfg_parallel_dummy_warms_supported_cfg_only_plan(self, monkeypatch): + def test_guidance_parallel_dummy_warms_full_guidance_plan(self, monkeypatch): from vllm_omni.diffusion.models.ltx2 import ltx2_runtime from vllm_omni.diffusion.models.ltx2.ltx2_recipes import LTX23_ONE_STAGE_RECIPE from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline - monkeypatch.setattr(ltx2_runtime, "get_classifier_free_guidance_world_size", lambda: 2) + monkeypatch.setattr(ltx2_runtime, "get_guidance_parallel_world_size", lambda: 2) pipe = object.__new__(LTX2Pipeline) req = SimpleNamespace(is_dummy_run=lambda: True) request_inputs = SimpleNamespace(guidance=LTX23_ONE_STAGE_RECIPE.request_guidance) - cfg_parallel_ready = pipe._setup_forward_runtime(req, request_inputs, attention_kwargs=None) + guidance_parallel_ready = pipe._setup_forward_runtime(req, request_inputs, attention_kwargs=None) - assert cfg_parallel_ready is True - assert pipe._guidance_plan.names == ("cond", "uncond") + assert guidance_parallel_ready is True + assert pipe._guidance_plan.names == ("cond", "uncond", "ptb", "mod") assert pipe._guidance_plan.spec.video.cfg_scale == 3.0 assert pipe._guidance_plan.spec.audio.cfg_scale == 7.0 + def test_guidance_parallel_warns_for_imbalanced_passes(self, monkeypatch): + from vllm_omni.diffusion.models.ltx2 import ltx2_guidance + + spec = ltx2_guidance.LTXGuidanceSpec( + video=ltx2_guidance.LTXModalityGuidance( + cfg_scale=3.0, + stg_scale=1.0, + stg_blocks=(1,), + ) + ) + plan = ltx2_guidance.LTXGuidancePlan.build(spec) + captured = [] + monkeypatch.setattr(ltx2_guidance, "get_guidance_parallel_rank", lambda: 0) + monkeypatch.setattr(ltx2_guidance.logger, "warning_once", lambda *args: captured.append(args)) + + ltx2_guidance.LTX_GUIDANCE_EXECUTOR.warn_if_imbalanced(plan, 4, "hq_generate") + + assert plan.names == ("cond", "uncond", "ptb") + assert len(captured) == 1 + assert captured[0][1:] == ("hq_generate", 3, 4, 1, 75.0) + + @pytest.mark.parametrize("guidance_rank", [0, 1]) + def test_default_guidance_is_sharded_and_combined(self, monkeypatch, guidance_rank): + from vllm_omni.diffusion.models.ltx2 import ltx2_guidance + from vllm_omni.diffusion.models.ltx2.ltx2_denoise import LTXDenoiseContext + from vllm_omni.diffusion.models.ltx2.ltx2_latents import LTXAVState + from vllm_omni.diffusion.models.ltx2.ltx2_recipes import LTX23_ONE_STAGE_RECIPE + + executor = ltx2_guidance.LTX_GUIDANCE_EXECUTOR + plan = ltx2_guidance.LTXGuidancePlan.build(LTX23_ONE_STAGE_RECIPE.request_guidance) + video_predictions = { + "cond": torch.tensor([[[0.2, -0.3]]]), + "uncond": torch.tensor([[[-0.4, 0.1]]]), + "ptb": torch.tensor([[[0.5, -0.2]]]), + "mod": torch.tensor([[[-0.1, 0.6]]]), + } + audio_predictions = { + "cond": torch.tensor([[[0.7, -0.2]]]), + "uncond": torch.tensor([[[0.1, 0.4]]]), + "ptb": torch.tensor([[[-0.3, 0.2]]]), + "mod": torch.tensor([[[0.4, -0.5]]]), + } + state = LTXAVState( + video=torch.tensor([[[1.0, -2.0]]]), + audio=torch.tensor([[[0.5, 3.0]]]), + ) + denoise_ctx = LTXDenoiseContext( + latents=state.video, + audio_latents=state.audio, + video_coords=torch.zeros(1, 1, 3), + audio_coords=torch.zeros(1, 1, 1), + ) + executor.prepare_denoise_context(plan, True, 2, denoise_ctx) + + class FakeGuidanceGroup: + def __init__(self): + self.call_index = 0 + + def all_gather(self, tensor, separate_tensors=True): + assert separate_tensors + gathered = ( + [video_predictions["cond"], video_predictions["uncond"]], + [video_predictions["ptb"], video_predictions["mod"]], + [audio_predictions["cond"], audio_predictions["uncond"]], + [audio_predictions["ptb"], audio_predictions["mod"]], + )[self.call_index] + torch.testing.assert_close(tensor, gathered[guidance_rank]) + self.call_index += 1 + return gathered + + group = FakeGuidanceGroup() + monkeypatch.setattr(ltx2_guidance, "get_guidance_parallel_world_size", lambda: 2) + monkeypatch.setattr(ltx2_guidance, "get_guidance_parallel_rank", lambda: guidance_rank) + monkeypatch.setattr(ltx2_guidance, "get_guidance_parallel_group", lambda: group) + + local_names = ("cond", "ptb") if guidance_rank == 0 else ("uncond", "mod") + captured = {} + + class FakeTransformer: + def __call__(self, **kwargs): + captured.update(kwargs) + return ( + torch.cat([video_predictions[name] for name in local_names]), + torch.cat([audio_predictions[name] for name in local_names]), + ) + + video_sigma = torch.tensor(0.25) + audio_sigma = torch.tensor(0.5) + + class FakePipeline: + scheduler = SimpleNamespace(sigmas=torch.stack([video_sigma])) + transformer = FakeTransformer() + + @staticmethod + def _transformer_cache_context(name): + assert name == "guided" + return nullcontext() + + @staticmethod + def _build_transformer_kwargs(forward_ctx, denoise_ctx, **kwargs): + del forward_ctx, denoise_ctx + return kwargs + + @staticmethod + def _video_guidance_model_sigma(sigma, denoise_ctx): + del denoise_ctx + return sigma + + positive_context = torch.ones(1, 1, 1) + negative_context = torch.zeros(1, 1, 1) + forward_ctx = SimpleNamespace( + prompt_context=SimpleNamespace( + positive_connector_prompt_embeds=positive_context, + positive_connector_audio_prompt_embeds=positive_context, + negative_connector_prompt_embeds=negative_context, + negative_connector_audio_prompt_embeds=negative_context, + ), + attention_kwargs=None, + audio_scheduler=SimpleNamespace(sigmas=torch.stack([audio_sigma])), + ) + + actual_video, actual_audio = executor.predict_parallel_guidance( + FakePipeline(), + plan, + index=0, + timestep=torch.tensor(1.0), + state=state, + forward_ctx=forward_ctx, + denoise_ctx=denoise_ctx, + ) + + expected_video = executor._guide_modality( + state.video, + video_predictions, + video_sigma, + plan.spec.video, + ) + expected_audio = executor._guide_modality( + state.audio, + audio_predictions, + audio_sigma, + plan.spec.audio, + ) + torch.testing.assert_close(actual_video, expected_video) + torch.testing.assert_close(actual_audio, expected_audio) + assert group.call_index == 4 + assert captured["hidden_states"].shape[0] == 2 + perturbations = captured["attention_kwargs"]["ltx_perturbation_kwargs"] + if guidance_rank == 0: + assert "video_self_attention_mask" in perturbations + assert "a2v_cross_attention_mask" not in perturbations + else: + assert "video_self_attention_mask" not in perturbations + assert "a2v_cross_attention_mask" in perturbations + class TestCFGParallelForwardPath: """Test the LTX-2.3 CFG-parallel denoising path without loading model weights.""" @@ -96,7 +252,7 @@ class TestCFGParallelForwardPath: pipe.tokenizer_max_length = 4 pipe.vae_spatial_compression_ratio = 32 pipe.vae_temporal_compression_ratio = 8 - monkeypatch.setattr(ltx2_runtime, "get_classifier_free_guidance_world_size", lambda: 1) + monkeypatch.setattr(ltx2_runtime, "get_guidance_parallel_world_size", lambda: 1) class StopAtEncodePromptError(Exception): pass @@ -238,10 +394,10 @@ class TestCFGParallelForwardPath: return [audio_pos, audio_neg] raise AssertionError(f"Unexpected gathered tensor: {tensor}") - monkeypatch.setattr(ltx2_runtime, "get_classifier_free_guidance_world_size", lambda: 2) - monkeypatch.setattr(ltx2_guidance, "get_classifier_free_guidance_world_size", lambda: 2) - monkeypatch.setattr(ltx2_guidance, "get_classifier_free_guidance_rank", lambda: cfg_rank) - monkeypatch.setattr(ltx2_guidance, "get_cfg_group", lambda: FakeCfgGroup()) + monkeypatch.setattr(ltx2_runtime, "get_guidance_parallel_world_size", lambda: 2) + monkeypatch.setattr(ltx2_guidance, "get_guidance_parallel_world_size", lambda: 2) + monkeypatch.setattr(ltx2_guidance, "get_guidance_parallel_rank", lambda: cfg_rank) + monkeypatch.setattr(ltx2_guidance, "get_guidance_parallel_group", lambda: FakeCfgGroup()) def fake_retrieve_timesteps(scheduler, num_inference_steps, device, timesteps, sigmas=None, mu=None): scheduler.sigmas = torch.tensor([0.25, 0.25], device=device) diff --git a/tests/entrypoints/test_serve.py b/tests/entrypoints/test_serve.py index cbe69302f..2606723f8 100644 --- a/tests/entrypoints/test_serve.py +++ b/tests/entrypoints/test_serve.py @@ -46,6 +46,16 @@ def test_serve_parser_accepts_strategy_config() -> None: assert args.get_explicit_kwargs_dict()["strategy_config"] == "/tmp/strategy.yaml" +def test_serve_parser_accepts_four_way_cfg_parallelism() -> None: + parser = TrackingArgumentParser() + subparsers = parser.add_subparsers(dest="subcommand") + OmniServeCommand().subparser_init(subparsers) + + args = parser.parse_args(["serve", "fake-model", "--omni", "--cfg-parallel-size", "4"]) + + assert args.cfg_parallel_size == 4 + + def _make_headless_args(**kwargs) -> TrackingNamespace: defaults = { "model": "fake-model", diff --git a/vllm_omni/config/omni_config.py b/vllm_omni/config/omni_config.py index dad16d184..422e8d784 100644 --- a/vllm_omni/config/omni_config.py +++ b/vllm_omni/config/omni_config.py @@ -368,7 +368,7 @@ class OmniStageDiffusionParallelConfig(OmniStageParallelConfig): ring_degree: int = Field(default=1, ge=1) allgather_degree: int = Field(default=1, ge=1) ulysses_mode: str = "strict" - cfg_parallel_size: int = Field(default=1, ge=1, le=3) + cfg_parallel_size: int = Field(default=1, ge=1) vae_patch_parallel_size: int = Field(default=1, ge=1) vae_parallel_mode: str = "tile" use_hsdp: bool = False diff --git a/vllm_omni/diffusion/data.py b/vllm_omni/diffusion/data.py index ca9d48d96..b97ffc311 100644 --- a/vllm_omni/diffusion/data.py +++ b/vllm_omni/diffusion/data.py @@ -185,7 +185,7 @@ class DiffusionParallelConfig: """ cfg_parallel_size: int = 1 - """Number of Classifier Free Guidance (CFG) parallel groups.""" + """Number of ranks used to execute guidance passes in parallel.""" vae_patch_parallel_size: int = 1 """Number of ranks used for VAE patch/tile parallelism (decode/encode).""" @@ -233,9 +233,6 @@ class DiffusionParallelConfig: assert self.ring_degree > 0, "Ring degree must be > 0" assert self.allgather_degree > 0, "AllGather degree must be > 0" assert self.cfg_parallel_size > 0, "CFG parallel size must be > 0" - assert self.cfg_parallel_size in [1, 2, 3], ( - f"CFG parallel size must be 1, 2, or 3, but got {self.cfg_parallel_size}" - ) assert self.vae_patch_parallel_size > 0, "VAE patch parallel size must be > 0" assert self.vae_parallel_mode in {"tile", "spatial_shard_height", "spatial_shard_width"}, ( "vae_parallel_mode must be one of {'tile', 'spatial_shard_height', 'spatial_shard_width'}, " diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py b/vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py index 6bd759057..f4d2f26f6 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py @@ -705,12 +705,12 @@ class LTXI2VConditioningMixin: latents=request_inputs.latents, ) - def _prepare_denoise_context_for_cfg( + def _prepare_denoise_context_for_guidance( self, forward_ctx: LTXForwardContext, denoise_ctx: LTXDenoiseContext, ) -> LTXDenoiseContext: - denoise_ctx = super()._prepare_denoise_context_for_cfg(forward_ctx, denoise_ctx) + denoise_ctx = super()._prepare_denoise_context_for_guidance(forward_ctx, denoise_ctx) if denoise_ctx.conditioning_mask is None: if self.unified_text_image_entry: return denoise_ctx @@ -755,9 +755,7 @@ class LTXI2VConditioningMixin: video_token_count=video_token_count, audio_token_count=audio_token_count, ) - conditioning_mask = ( - denoise_ctx.conditioning_mask if forward_ctx.cfg_parallel_ready else denoise_ctx.conditioning_mask_for_model - ) + conditioning_mask = denoise_ctx.conditioning_mask_for_model if conditioning_mask is None: raise ValueError("LTX I2V denoising requires a conditioning mask.") kwargs.update( diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_denoise.py b/vllm_omni/diffusion/models/ltx2/ltx2_denoise.py index 262c9334c..4a149be85 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_denoise.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_denoise.py @@ -33,7 +33,7 @@ class LTXForwardContext: request_inputs: LTXRequestInputs prompt_context: LTXPromptContext device: torch.device - cfg_parallel_ready: bool + guidance_parallel_ready: bool attention_kwargs: dict[str, Any] | None latent_num_frames: int latent_height: int @@ -358,9 +358,9 @@ def step_denoised_latents( do_true_cfg=pipeline.do_classifier_free_guidance, per_request_scheduler=forward_ctx.video_audio_step_adapter, ) - return pipeline._synchronize_cfg_parallel_step_output( + return pipeline._synchronize_guidance_parallel_step_output( latents, - do_true_cfg=pipeline.do_classifier_free_guidance, + guidance_parallel_ready=forward_ctx.guidance_parallel_ready, ) @@ -382,7 +382,7 @@ class LTXPhaseExecutor: prompt_context: LTXPromptContext | None = None, ) -> LTXPhaseResult: pipeline._check_forward_inputs(request_inputs, image=image) - cfg_parallel_ready = pipeline._setup_forward_runtime(req, request_inputs, attention_kwargs) + guidance_parallel_ready = pipeline._setup_forward_runtime(req, request_inputs, attention_kwargs) device = pipeline.device if prompt_context is None: prompt_context = pipeline._prepare_prompt_context( @@ -429,7 +429,7 @@ class LTXPhaseExecutor: request_inputs=request_inputs, prompt_context=prompt_context, device=device, - cfg_parallel_ready=cfg_parallel_ready, + guidance_parallel_ready=guidance_parallel_ready, attention_kwargs=attention_kwargs, latent_num_frames=latent_num_frames, latent_height=latent_height, @@ -449,7 +449,7 @@ class LTXPhaseExecutor: audio_coords=audio_coords, conditioning_mask=conditioning_mask, ) - denoise_ctx = pipeline._prepare_denoise_context_for_cfg(forward_ctx, denoise_ctx) + denoise_ctx = pipeline._prepare_denoise_context_for_guidance(forward_ctx, denoise_ctx) state = LTXDenoiseExecutor.run( pipeline, LTXAVState(video=denoise_ctx.latents, audio=denoise_ctx.audio_latents), diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_guidance.py b/vllm_omni/diffusion/models/ltx2/ltx2_guidance.py index 0b3d2dc2b..f35f5b998 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_guidance.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_guidance.py @@ -10,12 +10,19 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any import torch +from vllm.logger import init_logger from vllm_omni.diffusion.distributed.parallel_state import ( - get_cfg_group, - get_classifier_free_guidance_rank, - get_classifier_free_guidance_world_size, + get_cfg_group as get_guidance_parallel_group, ) +from vllm_omni.diffusion.distributed.parallel_state import ( + get_classifier_free_guidance_rank as get_guidance_parallel_rank, +) +from vllm_omni.diffusion.distributed.parallel_state import ( + get_classifier_free_guidance_world_size as get_guidance_parallel_world_size, +) + +logger = init_logger(__name__) if TYPE_CHECKING: from .ltx2_denoise import LTXDenoiseContext, LTXForwardContext @@ -110,10 +117,6 @@ class LTXGuidancePlan: def names(self) -> tuple[str, ...]: return tuple(item.name for item in self.passes) - @property - def cfg_parallel_compatible(self) -> bool: - return self.names == ("cond", "uncond") and not self.spec.do_rescale - def x0_from_velocity(sample: torch.Tensor, velocity: torch.Tensor, sigma: torch.Tensor) -> torch.Tensor: """Convert velocity to x0 using the official fp32 arithmetic order.""" @@ -212,23 +215,69 @@ class LTXGuidanceExecutor: """Build and execute official LTX multi-guidance Transformer passes.""" @staticmethod - def validate_cfg_world_size(plan: LTXGuidancePlan, cfg_world_size: int) -> None: - if cfg_world_size == 1: + def validate_guidance_world_size(plan: LTXGuidancePlan, guidance_world_size: int) -> None: + del plan + if guidance_world_size < 1: + raise ValueError(f"LTX guidance parallelism requires a positive parallel size, got {guidance_world_size}.") + + @staticmethod + def _parallel_assignments(pass_count: int, world_size: int) -> list[list[int]]: + assignments: list[list[int]] = [[] for _ in range(world_size)] + for pass_index in range(pass_count): + assignments[pass_index % world_size].append(pass_index) + return assignments + + @classmethod + def _model_pass_count( + cls, + plan: LTXGuidancePlan, + guidance_parallel_ready: bool, + guidance_world_size: int, + ) -> int: + if not guidance_parallel_ready: + return len(plan.passes) + assignments = cls._parallel_assignments(len(plan.passes), guidance_world_size) + return max(len(indices) for indices in assignments) + + @classmethod + def warn_if_imbalanced( + cls, + plan: LTXGuidancePlan, + guidance_world_size: int, + phase_name: str, + ) -> None: + pass_count = len(plan.passes) + if guidance_world_size <= 1 or pass_count % guidance_world_size == 0 or get_guidance_parallel_rank() != 0: return - if cfg_world_size != 2: - raise ValueError(f"LTX CFG parallelism supports cfg_parallel_size 1 or 2, got {cfg_world_size}.") - if not plan.cfg_parallel_compatible: - raise ValueError("LTX CFG parallelism only supports CFG-only guidance without rescale.") + slots_per_rank = math.ceil(pass_count / guidance_world_size) + total_slots = slots_per_rank * guidance_world_size + wasted_slots = total_slots - pass_count + utilization = 100.0 * pass_count / total_slots + logger.warning_once( + "LTX guidance parallelism is imbalanced for phase %r: %d guidance passes across %d ranks. " + "%d padded/discarded rank slot(s); expected guidance-slot utilization is %.1f%%.", + phase_name, + pass_count, + guidance_world_size, + wasted_slots, + utilization, + ) @staticmethod def prepare_denoise_context( plan: LTXGuidancePlan, - cfg_parallel_ready: bool, + guidance_parallel_ready: bool, + guidance_world_size: int, denoise_ctx: LTXDenoiseContext, ) -> LTXDenoiseContext: - if len(plan.passes) > 1 and not cfg_parallel_ready: - denoise_ctx.video_coords = _repeat_batch(denoise_ctx.video_coords, len(plan.passes)) - denoise_ctx.audio_coords = _repeat_batch(denoise_ctx.audio_coords, len(plan.passes)) + model_pass_count = LTXGuidanceExecutor._model_pass_count( + plan, + guidance_parallel_ready, + guidance_world_size, + ) + if model_pass_count > 1: + denoise_ctx.video_coords = _repeat_batch(denoise_ctx.video_coords, model_pass_count) + denoise_ctx.audio_coords = _repeat_batch(denoise_ctx.audio_coords, model_pass_count) return denoise_ctx @staticmethod @@ -265,7 +314,30 @@ class LTXGuidanceExecutor: ) return velocity_from_x0(sample, guided_x0, sigma) - def predict_parallel_cfg( + @staticmethod + def _guide_modality( + sample: torch.Tensor, + splits: dict[str, torch.Tensor], + sigma: torch.Tensor, + guidance: LTXModalityGuidance, + *, + model_sigma: torch.Tensor | None = None, + ) -> torch.Tensor: + model_sigma = sigma if model_sigma is None else model_sigma + # Official guidance reduces contiguous BSC tensors. The fused + # transformer output may be channel-major after splitting, which + # changes fp32 reduction order and can cross bf16 rounding bounds. + x0 = {name: x0_from_velocity(sample, value, model_sigma).contiguous() for name, value in splits.items()} + guided = combine_guided_x0( + cond=x0["cond"], + uncond_text=x0.get("uncond", 0.0), + uncond_perturbed=x0.get("ptb", 0.0), + uncond_modality=x0.get("mod", 0.0), + guidance=guidance, + ) + return velocity_from_x0(sample, guided, sigma) + + def predict_parallel_guidance( self, pipeline: Any, plan: LTXGuidancePlan, @@ -275,54 +347,93 @@ class LTXGuidanceExecutor: forward_ctx: LTXForwardContext, denoise_ctx: LTXDenoiseContext, ) -> tuple[torch.Tensor, torch.Tensor]: - self.validate_cfg_world_size(plan, get_classifier_free_guidance_world_size()) + guidance_world_size = get_guidance_parallel_world_size() + self.validate_guidance_world_size(plan, guidance_world_size) + guidance_rank = get_guidance_parallel_rank() + assignments = self._parallel_assignments(len(plan.passes), guidance_world_size) + local_pass_indices = assignments[guidance_rank] + model_pass_count = max(len(indices) for indices in assignments) + + # Every rank executes and gathers the same number of slots. Shorter + # assignments use a conditional pass whose output is discarded. + padded_pass_indices: list[int | None] = local_pass_indices + [None] * ( + model_pass_count - len(local_pass_indices) + ) + local_passes = tuple( + plan.passes[0] if pass_index is None else plan.passes[pass_index] for pass_index in padded_pass_indices + ) + local_plan = LTXGuidancePlan(spec=plan.spec, passes=local_passes) + prompt = forward_ctx.prompt_context - negative = get_classifier_free_guidance_rank() == 1 - video_context = ( - prompt.negative_connector_prompt_embeds - if negative and plan.spec.video.do_cfg - else prompt.positive_connector_prompt_embeds + video_contexts: list[torch.Tensor] = [] + audio_contexts: list[torch.Tensor] = [] + for denoise_pass in local_passes: + video_context = ( + prompt.negative_connector_prompt_embeds + if denoise_pass.negative_video_context + else prompt.positive_connector_prompt_embeds + ) + audio_context = ( + prompt.negative_connector_audio_prompt_embeds + if denoise_pass.negative_audio_context + else prompt.positive_connector_audio_prompt_embeds + ) + if video_context is None or audio_context is None: + raise ValueError("Negative prompt context is required when LTX CFG is enabled.") + video_contexts.append(video_context) + audio_contexts.append(audio_context) + + video_input = _repeat_batch(state.video, model_pass_count).to(prompt.positive_connector_prompt_embeds.dtype) + audio_input = _repeat_batch(state.audio, model_pass_count).to( + prompt.positive_connector_audio_prompt_embeds.dtype ) - audio_context = ( - prompt.negative_connector_audio_prompt_embeds - if negative and plan.spec.audio.do_cfg - else prompt.positive_connector_audio_prompt_embeds - ) - if video_context is None or audio_context is None: - raise ValueError("Negative prompt context is required when LTX CFG is enabled.") - video_input = state.video.to(prompt.positive_connector_prompt_embeds.dtype) - audio_input = state.audio.to(prompt.positive_connector_audio_prompt_embeds.dtype) + attention_kwargs = dict(forward_ctx.attention_kwargs or {}) + perturbations = build_perturbation_kwargs(local_plan, state.video.shape[0], video_input) + if perturbations: + attention_kwargs["ltx_perturbation_kwargs"] = perturbations kwargs = pipeline._build_transformer_kwargs( forward_ctx, denoise_ctx, hidden_states=video_input, audio_hidden_states=audio_input, - encoder_hidden_states=video_context, - audio_encoder_hidden_states=audio_context, + encoder_hidden_states=torch.cat(video_contexts), + audio_encoder_hidden_states=torch.cat(audio_contexts), encoder_attention_mask=None, audio_encoder_attention_mask=None, ts=timestep.expand(video_input.shape[0]), + attention_kwargs=attention_kwargs, ) - local_video, local_audio = pipeline.predict_noise(**kwargs) - group = get_cfg_group() - video = group.all_gather(local_video, separate_tensors=True) - audio = group.all_gather(local_audio, separate_tensors=True) + with pipeline._transformer_cache_context("guided"): + local_video, local_audio = pipeline.transformer(**kwargs) + local_video_slots = local_video.chunk(model_pass_count) + local_audio_slots = local_audio.chunk(model_pass_count) + + group = get_guidance_parallel_group() + gathered_video_slots = [group.all_gather(value, separate_tensors=True) for value in local_video_slots] + gathered_audio_slots = [group.all_gather(value, separate_tensors=True) for value in local_audio_slots] + + video_splits: dict[str, torch.Tensor] = {} + audio_splits: dict[str, torch.Tensor] = {} + for pass_index, denoise_pass in enumerate(plan.passes): + owner_rank = pass_index % guidance_world_size + slot_index = pass_index // guidance_world_size + video_splits[denoise_pass.name] = gathered_video_slots[slot_index][owner_rank] + audio_splits[denoise_pass.name] = gathered_audio_slots[slot_index][owner_rank] + video_sigma = pipeline.scheduler.sigmas[index] return ( - self.combine_cfg_velocity( + self._guide_modality( state.video, - video[0], - video[1], + video_splits, video_sigma, - plan.spec.video.cfg_scale, + plan.spec.video, model_sigma=pipeline._video_guidance_model_sigma(video_sigma, denoise_ctx), ), - self.combine_cfg_velocity( + self._guide_modality( state.audio, - audio[0], - audio[1], + audio_splits, forward_ctx.audio_scheduler.sigmas[index], - plan.spec.audio.cfg_scale, + plan.spec.audio, ), ) @@ -336,8 +447,8 @@ class LTXGuidanceExecutor: forward_ctx: LTXForwardContext, denoise_ctx: LTXDenoiseContext, ) -> tuple[torch.Tensor, torch.Tensor]: - if forward_ctx.cfg_parallel_ready: - return self.predict_parallel_cfg(pipeline, plan, index, timestep, state, forward_ctx, denoise_ctx) + if forward_ctx.guidance_parallel_ready: + return self.predict_parallel_guidance(pipeline, plan, index, timestep, state, forward_ctx, denoise_ctx) prompt = forward_ctx.prompt_context video_contexts: list[torch.Tensor] = [] @@ -382,30 +493,8 @@ class LTXGuidanceExecutor: video_splits = dict(zip(plan.names, video_velocity.chunk(pass_count), strict=True)) audio_splits = dict(zip(plan.names, audio_velocity.chunk(pass_count), strict=True)) - def guide_modality( - sample: torch.Tensor, - splits: dict[str, torch.Tensor], - sigma: torch.Tensor, - guidance: LTXModalityGuidance, - *, - model_sigma: torch.Tensor | None = None, - ) -> torch.Tensor: - model_sigma = sigma if model_sigma is None else model_sigma - # Official guidance reduces contiguous BSC tensors. The fused - # transformer output may be channel-major after splitting, which - # changes fp32 reduction order and can cross bf16 rounding bounds. - x0 = {name: x0_from_velocity(sample, value, model_sigma).contiguous() for name, value in splits.items()} - guided = combine_guided_x0( - cond=x0["cond"], - uncond_text=x0.get("uncond", 0.0), - uncond_perturbed=x0.get("ptb", 0.0), - uncond_modality=x0.get("mod", 0.0), - guidance=guidance, - ) - return velocity_from_x0(sample, guided, sigma) - return ( - guide_modality( + self._guide_modality( state.video, video_splits, pipeline.scheduler.sigmas[index], @@ -415,7 +504,7 @@ class LTXGuidanceExecutor: denoise_ctx, ), ), - guide_modality( + self._guide_modality( state.audio, audio_splits, forward_ctx.audio_scheduler.sigmas[index], diff --git a/vllm_omni/diffusion/models/ltx2/ltx2_runtime.py b/vllm_omni/diffusion/models/ltx2/ltx2_runtime.py index b5bb18a9b..4b74ebcb8 100644 --- a/vllm_omni/diffusion/models/ltx2/ltx2_runtime.py +++ b/vllm_omni/diffusion/models/ltx2/ltx2_runtime.py @@ -17,7 +17,9 @@ from vllm.model_executor.models.utils import AutoWeightsLoader from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin -from vllm_omni.diffusion.distributed.parallel_state import get_classifier_free_guidance_world_size +from vllm_omni.diffusion.distributed.parallel_state import ( + get_classifier_free_guidance_world_size as get_guidance_parallel_world_size, +) from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin @@ -44,8 +46,6 @@ from .ltx2_guidance import ( LTX_GUIDANCE_EXECUTOR, LTXGuidanceExecutor, LTXGuidancePlan, - LTXGuidanceSpec, - LTXModalityGuidance, ) from .ltx2_recipes import ( LTXPhaseRecipe, @@ -259,7 +259,7 @@ class LTXRuntime( def _enter_phase(self, phase: LTXPhaseRecipe) -> None: """Hook for a future phase-weight strategy.""" - del phase + self._active_phase_name = phase.name def prepare_latents( self, @@ -323,6 +323,10 @@ class LTXRuntime( def do_classifier_free_guidance(self): return self._guidance_plan.spec.do_cfg + @property + def do_guidance(self): + return len(self._guidance_plan.passes) > 1 + @property def interrupt(self): return self._interrupt @@ -372,16 +376,16 @@ class LTXRuntime( ), ) - def _synchronize_cfg_parallel_step_output( + def _synchronize_guidance_parallel_step_output( self, latents: tuple[torch.Tensor, torch.Tensor], - do_true_cfg: bool, + guidance_parallel_ready: bool, ) -> tuple[torch.Tensor, torch.Tensor]: - if not (do_true_cfg and get_classifier_free_guidance_world_size() > 1): + if not (guidance_parallel_ready and get_guidance_parallel_world_size() > 1): return latents # CUDA async execution otherwise permits numerical drift to accumulate - # across CFG-parallel denoise steps. + # across guidance-parallel denoise steps. latents = tuple(tensor.contiguous() for tensor in latents) current_omni_platform.synchronize() return latents @@ -393,26 +397,16 @@ class LTXRuntime( attention_kwargs: dict[str, Any] | None, ) -> bool: self._guidance_plan = LTXGuidancePlan.build(request_inputs.guidance) - del attention_kwargs + del req, attention_kwargs self._interrupt = False - cfg_world_size = get_classifier_free_guidance_world_size() - if ( - req.is_dummy_run() - and cfg_world_size == 2 - and self.do_classifier_free_guidance - and not self._guidance_plan.cfg_parallel_compatible - ): - # The single-device warmup follows the full recipe. CFG2 executes - # the same per-modality CFG scales without STG/modality passes. - self._guidance_plan = LTXGuidancePlan.build( - LTXGuidanceSpec( - video=LTXModalityGuidance(cfg_scale=self._guidance_plan.spec.video.cfg_scale), - audio=LTXModalityGuidance(cfg_scale=self._guidance_plan.spec.audio.cfg_scale), - ) - ) - if self.do_classifier_free_guidance: - self.guidance_executor.validate_cfg_world_size(self._guidance_plan, cfg_world_size) - return self.do_classifier_free_guidance and cfg_world_size > 1 + guidance_world_size = get_guidance_parallel_world_size() + self.guidance_executor.validate_guidance_world_size(self._guidance_plan, guidance_world_size) + self.guidance_executor.warn_if_imbalanced( + self._guidance_plan, + guidance_world_size, + getattr(self, "_active_phase_name", "generate"), + ) + return self.do_guidance and guidance_world_size > 1 def _check_forward_inputs( self, @@ -594,14 +588,15 @@ class LTXRuntime( padded_length = latent_ops.get_sp_padded_audio_latent_length(requested_length, int(sp_size)) return requested_length if provided_length in {requested_length, padded_length} else provided_length - def _prepare_denoise_context_for_cfg( + def _prepare_denoise_context_for_guidance( self, forward_ctx: LTXForwardContext, denoise_ctx: LTXDenoiseContext, ) -> LTXDenoiseContext: return self.guidance_executor.prepare_denoise_context( self._guidance_plan, - forward_ctx.cfg_parallel_ready, + forward_ctx.guidance_parallel_ready, + get_guidance_parallel_world_size(), denoise_ctx, ) diff --git a/vllm_omni/entrypoints/cli/serve.py b/vllm_omni/entrypoints/cli/serve.py index 8f6a88190..83e975528 100644 --- a/vllm_omni/entrypoints/cli/serve.py +++ b/vllm_omni/entrypoints/cli/serve.py @@ -678,8 +678,7 @@ class OmniServeCommand(CLISubcommand): "--cfg-parallel-size", type=int, default=1, - choices=[1, 2], - help="Number of devices for CFG parallel computation for diffusion models. " + help="Number of devices used to execute diffusion guidance passes in parallel. " "Equivalent to setting DiffusionParallelConfig.cfg_parallel_size.", ) omni_config_group.add_argument( From aa5364f70592181c1538fe3fd2235d5f4f7a5e66 Mon Sep 17 00:00:00 2001 From: y-null Date: Thu, 30 Jul 2026 09:56:19 +0800 Subject: [PATCH 27/37] [refactor][MiniCPM-o] Consolidate MiniCPM-o 4.5 2-GPU deploy configs and E2E tests (#5458) Signed-off-by: y_null Signed-off-by: amy-why-3459 Co-authored-by: amy-why-3459 Co-authored-by: Hongsheng Liu Co-authored-by: Gao Han --- examples/online_serving/minicpmo/README.md | 5 +- recipes/OpenBMB/MiniCPM-o-4_5.md | 6 +- .../test_invalid_minicpmo_4_5_omni.py | 10 ++- .../offline_inference/test_minicpmo_4_5.py | 18 ++-- tests/e2e/online_serving/test_minicpmo_4_5.py | 36 ++++---- .../test_minicpmo_4_5_expansion.py | 12 +-- .../models/minicpmo_4_5/test_pipeline.py | 3 +- vllm_omni/deploy/minicpmo_4_5_2gpu.yaml | 90 ++++++++++++++++++- vllm_omni/deploy/minicpmo_4_5_batching.yaml | 89 ------------------ 9 files changed, 132 insertions(+), 137 deletions(-) delete mode 100644 vllm_omni/deploy/minicpmo_4_5_batching.yaml diff --git a/examples/online_serving/minicpmo/README.md b/examples/online_serving/minicpmo/README.md index 134ddd62c..e565a3de3 100644 --- a/examples/online_serving/minicpmo/README.md +++ b/examples/online_serving/minicpmo/README.md @@ -25,15 +25,14 @@ The deploy config auto-loads via `--omni`. The default `vllm_omni/deploy/minicpmo_4_5.yaml` keeps all three stages on logical device 0 with memory budgets of 55%, 22%, and 22%. The profile admits at most four sequences per stage and bounds startup video profiling to 32 -frames per video. For throughput,`minicpmo_4_5_batching.yaml` gives the Thinker +frames per video. For throughput,`minicpmo_4_5_2gpu.yaml` gives the Thinker GPU 0 (90%) and colocates the Talker (55%) and Code2Wav (35%) on GPU 1. That profile admits at most four concurrent sequences per stage. | deploy config | GPUs | Notes | |---|---|---| | `minicpmo_4_5.yaml` (default) | 1 | Memory-constrained compatibility layout. | -| `minicpmo_4_5_batching.yaml` | 2 | Recommended continuous-batching layout; Talker and Code2Wav share GPU 1. | -| `minicpmo_4_5_2gpu.yaml` | 2 | Backward-compatible alias for `minicpmo_4_5_batching.yaml`. | +| `minicpmo_4_5_2gpu.yaml` | 2 | Recommended continuous-batching layout; Talker and Code2Wav share GPU 1. | | `minicpmo_4_5_3gpu.yaml` | 3 | One GPU per stage. | | `minicpmo_4_5_8x4090.yaml` | 8 | Full 8x4090 layout. | | `minicpmo_4_5_duplex.yaml` | 1 | Experimental native full-duplex overlay. | diff --git a/recipes/OpenBMB/MiniCPM-o-4_5.md b/recipes/OpenBMB/MiniCPM-o-4_5.md index 97b3c6d6d..87f68af5d 100644 --- a/recipes/OpenBMB/MiniCPM-o-4_5.md +++ b/recipes/OpenBMB/MiniCPM-o-4_5.md @@ -30,8 +30,6 @@ also provided. `hf_config.version="4.5"`): - Default single-GPU compatibility layout (auto-loaded): [`vllm_omni/deploy/minicpmo_4_5.yaml`](../../vllm_omni/deploy/minicpmo_4_5.yaml) - - Recommended 2-GPU continuous-batching layout: - [`vllm_omni/deploy/minicpmo_4_5_batching.yaml`](../../vllm_omni/deploy/minicpmo_4_5_batching.yaml) - 2-GPU and 3-GPU layouts: [`vllm_omni/deploy/minicpmo_4_5_2gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_2gpu.yaml), [`vllm_omni/deploy/minicpmo_4_5_3gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_3gpu.yaml) @@ -68,7 +66,7 @@ Code2Wav consumes them through a shared-memory async connector. MiniCPM-o 4.5 now requires the three-stage topology: the Talker owns request-local codec generation and Code2Wav owns waveform state and reference-voice prompt features. `minicpmo_4_5.yaml` remains the stable -single-GPU entry point; `minicpmo_4_5_batching.yaml` is the recommended +single-GPU entry point; `minicpmo_4_5_2gpu.yaml` is the recommended two-GPU profile. The removed fused two-stage implementation is not retained as a fallback because it would duplicate state machines and correctness paths. @@ -105,7 +103,7 @@ The deploy config is auto-loaded by the model registry — no For the recommended two-GPU layout, add: ```bash ---deploy-config vllm_omni/deploy/minicpmo_4_5_batching.yaml +--deploy-config vllm_omni/deploy/minicpmo_4_5_2gpu.yaml ``` #### Performance comparison diff --git a/tests/dfx/reliability/invalid_param_test/test_invalid_minicpmo_4_5_omni.py b/tests/dfx/reliability/invalid_param_test/test_invalid_minicpmo_4_5_omni.py index 2082be10d..9c6880d68 100644 --- a/tests/dfx/reliability/invalid_param_test/test_invalid_minicpmo_4_5_omni.py +++ b/tests/dfx/reliability/invalid_param_test/test_invalid_minicpmo_4_5_omni.py @@ -14,15 +14,17 @@ from tests.helpers.stage_config import get_deploy_config_path pytestmark = [pytest.mark.slow, pytest.mark.omni, pytest.mark.full_model] -_MINICPMO_DEPLOY = get_deploy_config_path("minicpmo_4_5_batching.yaml") +_MINICPMO_DEPLOY = get_deploy_config_path("minicpmo_4_5.yaml") _MINICPMO_SERVER = [ pytest.param( OmniServerParams( model="openbmb/MiniCPM-o-4_5", stage_config_path=_MINICPMO_DEPLOY, - use_stage_cli=True, - server_args=["--trust-remote-code"], + use_stage_cli=False, + server_args=[ + "--trust-remote-code", + ], ), id="minicpmo_4_5", ) @@ -48,7 +50,7 @@ def _prompt() -> str: return "What is the capital of China? Answer in 20 words." -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", _MINICPMO_SERVER, indirect=True) def test_invalid_audio_format_rejected(omni_server: OmniServerParams, openai_client: OpenAIClientHandler) -> None: """ diff --git a/tests/e2e/offline_inference/test_minicpmo_4_5.py b/tests/e2e/offline_inference/test_minicpmo_4_5.py index a8855e04b..12f24e3dc 100644 --- a/tests/e2e/offline_inference/test_minicpmo_4_5.py +++ b/tests/e2e/offline_inference/test_minicpmo_4_5.py @@ -14,7 +14,7 @@ from tests.helpers.stage_config import get_deploy_config_path models = ["openbmb/MiniCPM-o-4_5"] -_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5_batching.yaml") +_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5.yaml") test_params = [(model, None, {"deploy_config": _CI_DEPLOY, "trust_remote_code": True}) for model in models] @@ -31,11 +31,10 @@ def get_question(prompt_type: str = "text") -> str: return prompts.get(prompt_type, prompts["text"]) -@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/5437") @pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) def test_text_to_text(omni_runner, omni_runner_handler) -> None: """Test processing text, generating text output.""" @@ -45,7 +44,7 @@ def test_text_to_text(omni_runner, omni_runner_handler) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) def test_audio_to_text(omni_runner, omni_runner_handler) -> None: """Test processing audio, generating text output.""" @@ -58,7 +57,7 @@ def test_audio_to_text(omni_runner, omni_runner_handler) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) def test_image_to_text(omni_runner, omni_runner_handler) -> None: """Test processing image, generating text output.""" @@ -69,7 +68,7 @@ def test_image_to_text(omni_runner, omni_runner_handler) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) def test_video_to_text(omni_runner, omni_runner_handler) -> None: """Test processing video, generating text output.""" @@ -80,7 +79,7 @@ def test_video_to_text(omni_runner, omni_runner_handler) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) def test_text_to_audio(omni_runner, omni_runner_handler) -> None: """Test processing text and generating audio through Talker and Code2Wav.""" @@ -88,11 +87,10 @@ def test_text_to_audio(omni_runner, omni_runner_handler) -> None: omni_runner_handler.send_omni_request(request_config) -@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/5437") @pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) def test_mix_to_audio(omni_runner, omni_runner_handler) -> None: """Test processing mixed modalities (image + audio), generating audio output.""" @@ -111,7 +109,7 @@ def test_mix_to_audio(omni_runner, omni_runner_handler) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) def test_video_to_audio(omni_runner, omni_runner_handler) -> None: """Test processing video, generating audio output.""" diff --git a/tests/e2e/online_serving/test_minicpmo_4_5.py b/tests/e2e/online_serving/test_minicpmo_4_5.py index a4fefec98..7e77de2e1 100644 --- a/tests/e2e/online_serving/test_minicpmo_4_5.py +++ b/tests/e2e/online_serving/test_minicpmo_4_5.py @@ -1,7 +1,10 @@ -"""E2E online tests for MiniCPM-o 4.5 multimodal input and audio/text output. +""" +E2E Online tests for MiniCPM-o 4.5 model with multimodal input and audio / text output. -The batching deployment uses async chunk transfer across separate Thinker, -Talker, and Code2Wav stages. +MiniCPM-o 4.5 is a three-stage pipeline: Stage 0 Thinker (text), Stage 1 Talker +(TTS codec), Stage 2 Code2Wav (vocoder). The talker -> Code2Wav bridge supports +both async_chunk (streaming) and full-payload (``--no-async-chunk``) modes, so the +tests below exercise the sync path as well. """ import os @@ -16,15 +19,17 @@ from tests.helpers.stage_config import get_deploy_config_path os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" _MODEL = "openbmb/MiniCPM-o-4_5" -_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5_batching.yaml") +_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5.yaml") test_params = [ pytest.param( OmniServerParams( model=_MODEL, stage_config_path=_CI_DEPLOY, - use_stage_cli=True, - server_args=["--trust-remote-code"], + use_stage_cli=False, + server_args=[ + "--trust-remote-code", + ], ), id="default", ) @@ -60,11 +65,10 @@ def get_max_batch_size(size_type="few"): return batch_sizes.get(size_type, 5) -@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/5437") @pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_to_text_001(omni_server, openai_client) -> None: """ @@ -89,12 +93,12 @@ def test_text_to_text_001(omni_server, openai_client) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_to_audio_001(omni_server, openai_client) -> None: """ Test text-only input generating text + audio output via OpenAI API. - This exercises Talker TTS region detection and the Code2Wav stage. + This exercises the talker TTS region detection and in-process token2wav vocoder. Deploy Setting: default 2GPU Input Modal: text Output Modal: text + audio @@ -114,7 +118,7 @@ def test_text_to_audio_001(omni_server, openai_client) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_audio_to_text_audio_001(omni_server, openai_client) -> None: """ @@ -128,14 +132,13 @@ def test_audio_to_text_audio_001(omni_server, openai_client) -> None: messages = dummy_messages_from_mix_data( system_prompt=get_system_prompt(), audio_data_url=audio_data_url, - content_text=get_prompt(), + content_text=get_prompt("mix"), ) request_config = { "model": omni_server.model, "messages": messages, "stream": True, - "key_words": {"text": ["Beijing"]}, } openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) @@ -143,7 +146,7 @@ def test_audio_to_text_audio_001(omni_server, openai_client) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_image_to_text_audio_001(omni_server, openai_client) -> None: """ @@ -171,7 +174,7 @@ def test_image_to_text_audio_001(omni_server, openai_client) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_video_to_text_audio_001(omni_server, openai_client) -> None: """ @@ -197,11 +200,10 @@ def test_video_to_text_audio_001(omni_server, openai_client) -> None: openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) -@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/5437") @pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_mix_to_text_audio_001(omni_server, openai_client) -> None: """ diff --git a/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py b/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py index 068502903..089a304a7 100644 --- a/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py +++ b/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py @@ -18,14 +18,14 @@ pytestmark = [pytest.mark.full_model, pytest.mark.omni] os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" _MODEL = "openbmb/MiniCPM-o-4_5" -_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5_batching.yaml") +_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5.yaml") test_params = [ pytest.param( OmniServerParams( model=_MODEL, stage_config_path=_CI_DEPLOY, - use_stage_cli=True, + use_stage_cli=False, server_args=[ "--trust-remote-code", ], @@ -63,7 +63,7 @@ def get_max_batch_size(size_type="few"): return batch_sizes.get(size_type, 5) -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_video_to_text_001(omni_server, openai_client) -> None: """ @@ -90,7 +90,7 @@ def test_text_video_to_text_001(omni_server, openai_client) -> None: openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_sequential_requests_independent(omni_server, openai_client) -> None: """ @@ -131,7 +131,7 @@ def test_sequential_requests_independent(omni_server, openai_client) -> None: ) -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_to_audio_long_output_001(omni_server, openai_client) -> None: """ @@ -157,7 +157,7 @@ def test_text_to_audio_long_output_001(omni_server, openai_client) -> None: openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_chinese_text_to_audio(omni_server, openai_client) -> None: """ diff --git a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py index 2442cad44..b6f11ff5d 100644 --- a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py +++ b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py @@ -36,7 +36,6 @@ _PIPELINE_KEY = "minicpmo_4_5" _DEPLOY_DIR = Path(__file__).resolve().parents[4] / "vllm_omni" / "deploy" _DEPLOY_LAYOUTS = { "minicpmo_4_5.yaml": ["0", "0", "0"], - "minicpmo_4_5_batching.yaml": ["0", "1", "1"], "minicpmo_4_5_2gpu.yaml": ["0", "1", "1"], "minicpmo_4_5_3gpu.yaml": ["0", "1", "2"], "minicpmo_4_5_8x4090.yaml": ["0,1,2,3", "4", "5"], @@ -170,7 +169,7 @@ class TestDeployTopology: 0.22, ] assert stages[0].yaml_engine_args["limit_mm_per_prompt"] == {"video": {"count": 1, "num_frames": 32}} - elif filename in {"minicpmo_4_5_batching.yaml", "minicpmo_4_5_2gpu.yaml"}: + elif filename in {"minicpmo_4_5_2gpu.yaml"}: assert [stage.yaml_engine_args["gpu_memory_utilization"] for stage in stages] == [ 0.9, 0.55, diff --git a/vllm_omni/deploy/minicpmo_4_5_2gpu.yaml b/vllm_omni/deploy/minicpmo_4_5_2gpu.yaml index eddfd0877..75693feb7 100644 --- a/vllm_omni/deploy/minicpmo_4_5_2gpu.yaml +++ b/vllm_omni/deploy/minicpmo_4_5_2gpu.yaml @@ -1,2 +1,88 @@ -# Backward-compatible name for the canonical two-GPU batching profile. -base_config: minicpmo_4_5_batching.yaml +# MiniCPM-o 4.5 two-GPU continuous-batching layout. +# +# Thinker owns GPU 0. Talker and Code2Wav share GPU 1, keeping text decoding +# isolated from codec/vocoder contention while preserving stage-2 batching. +pipeline: minicpmo_4_5 +async_chunk: true + +trust_remote_code: true +enable_prefix_caching: false + +connectors: + connector_of_shared_memory: + name: SharedMemoryConnector + extra: + connector_get_sleep_s: 0.01 + connector_get_max_wait_first_chunk: 3000 + connector_get_max_wait: 300 + codec_chunk_frames: 25 + codec_left_context_frames: 3 + +stages: + - stage_id: 0 + max_num_seqs: 4 + gpu_memory_utilization: 0.9 + enforce_eager: false + tensor_parallel_size: 1 + max_num_batched_tokens: 16384 + devices: "0" + limit_mm_per_prompt: + video: 1 + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 2048 + seed: 42 + detokenize: true + repetition_penalty: 1.1 + + - stage_id: 1 + max_num_seqs: 4 + gpu_memory_utilization: 0.55 + enforce_eager: false + max_num_batched_tokens: 8192 + devices: "1" + output_connectors: + to_stage_2: connector_of_shared_memory + # Codec sampling is model-owned by the checkpoint's tts_config (defaults: + # T=0.8, top-k=25, top-p=0.85, repetition penalty=1.05, seed=42). + # These parameters only govern vLLM's binary continue/stop token. + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + min_tokens: 50 + max_tokens: 2048 + stop_token_ids: [1] + seed: 42 + detokenize: false + + - stage_id: 2 + max_num_seqs: 4 + gpu_memory_utilization: 0.35 + enforce_eager: true + enable_chunked_prefill: false + max_num_batched_tokens: 65536 + max_model_len: 65536 + devices: "1" + input_connectors: + from_stage_1: connector_of_shared_memory + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 65536 + detokenize: true + +platforms: + npu: + stages: + - stage_id: 0 + max_num_batched_tokens: 8192 + compilation_config: + cudagraph_mode: PIECEWISE + - stage_id: 1 + max_num_batched_tokens: 8192 + compilation_config: + cudagraph_mode: PIECEWISE diff --git a/vllm_omni/deploy/minicpmo_4_5_batching.yaml b/vllm_omni/deploy/minicpmo_4_5_batching.yaml deleted file mode 100644 index 7e6b1fb63..000000000 --- a/vllm_omni/deploy/minicpmo_4_5_batching.yaml +++ /dev/null @@ -1,89 +0,0 @@ -# MiniCPM-o 4.5 two-GPU continuous-batching layout. -# -# Thinker owns GPU 0. Talker and Code2Wav share GPU 1, keeping text decoding -# isolated from codec/vocoder contention while preserving stage-2 batching. -pipeline: minicpmo_4_5 -async_chunk: true - -trust_remote_code: true -enable_prefix_caching: false - -connectors: - connector_of_shared_memory: - name: SharedMemoryConnector - extra: - connector_get_sleep_s: 0.01 - connector_get_max_wait_first_chunk: 3000 - connector_get_max_wait: 300 - codec_chunk_frames: 25 - codec_left_context_frames: 3 - -stages: - - stage_id: 0 - max_num_seqs: 4 - gpu_memory_utilization: 0.9 - enforce_eager: false - tensor_parallel_size: 1 - max_num_batched_tokens: 16384 - devices: "0" - limit_mm_per_prompt: - video: 1 - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 2048 - seed: 42 - detokenize: true - repetition_penalty: 1.1 - - - stage_id: 1 - max_num_seqs: 4 - gpu_memory_utilization: 0.55 - enforce_eager: false - max_num_batched_tokens: 8192 - devices: "1" - output_connectors: - to_stage_2: connector_of_shared_memory - # Codec sampling is model-owned by the checkpoint's tts_config (defaults: - # T=0.8, top-k=25, top-p=0.85, repetition penalty=1.05, seed=42). - # These parameters only govern vLLM's binary continue/stop token. - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - min_tokens: 50 - max_tokens: 2048 - stop_token_ids: [1] - seed: 42 - detokenize: false - - - stage_id: 2 - max_num_seqs: 4 - gpu_memory_utilization: 0.35 - enforce_eager: true - enable_chunked_prefill: false - max_num_batched_tokens: 65536 - max_model_len: 65536 - devices: "1" - input_connectors: - from_stage_1: connector_of_shared_memory - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 65536 - detokenize: true - -platforms: - npu: - stages: - - stage_id: 0 - max_num_batched_tokens: 8192 - compilation_config: - cudagraph_mode: PIECEWISE - - stage_id: 1 - max_num_batched_tokens: 8192 - gpu_memory_utilization: 0.4 - compilation_config: - cudagraph_mode: PIECEWISE From 0461da345a61fac25bce467ee6a1b3e43ca0ae7e Mon Sep 17 00:00:00 2001 From: "ruirui(rein) yang" <73573651+R2-Y@users.noreply.github.com> Date: Thu, 30 Jul 2026 09:59:44 +0800 Subject: [PATCH 28/37] [bugfix][MiniCPM-o]delete duplicate files (#5535) Signed-off-by: R2-Y Co-authored-by: Gao Han --- examples/online_serving/minicpmo/README.md | 15 ++-- .../minicpmo/streaming_chat_completion.py | 71 ------------------- recipes/OpenBMB/MiniCPM-o-4_5.md | 14 ++-- .../online_serving/test_minicpmo_streaming.py | 29 -------- 4 files changed, 16 insertions(+), 113 deletions(-) delete mode 100644 examples/online_serving/minicpmo/streaming_chat_completion.py delete mode 100644 tests/examples/online_serving/test_minicpmo_streaming.py diff --git a/examples/online_serving/minicpmo/README.md b/examples/online_serving/minicpmo/README.md index e565a3de3..94fae5fa7 100644 --- a/examples/online_serving/minicpmo/README.md +++ b/examples/online_serving/minicpmo/README.md @@ -88,17 +88,18 @@ python openai_chat_completion_client_for_multimodal_generation.py \ --prompt "Briefly introduce yourself." ``` -Streaming text + audio: +Streaming text + audio (use `--stream`): ```bash -python streaming_chat_completion.py \ - --base-url http://localhost:8099/v1 \ - --output minicpmo_stream.wav +python openai_chat_completion_client_for_multimodal_generation.py \ + --query-type text \ + --prompt "Briefly introduce yourself." \ + --port 8099 \ + --stream ``` -The example prints text deltas immediately and joins the independently encoded -24 kHz WAV audio deltas into one valid WAV file. Add `--text-only` to skip -Talker and Code2Wav. +The client prints text deltas as they arrive and saves streamed audio chunks +to WAV files. Shared helpers also work if you pass MiniCPM defaults yourself: diff --git a/examples/online_serving/minicpmo/streaming_chat_completion.py b/examples/online_serving/minicpmo/streaming_chat_completion.py deleted file mode 100644 index b4a4be2f0..000000000 --- a/examples/online_serving/minicpmo/streaming_chat_completion.py +++ /dev/null @@ -1,71 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Stream MiniCPM-o 4.5 text and audio from /v1/chat/completions.""" - -from __future__ import annotations - -import argparse -import base64 -import io -import wave -from pathlib import Path - -from openai import OpenAI - - -def append_wav_chunk(encoded: str, pcm_parts: list[bytes]) -> tuple[int, int, int]: - """Decode one base64 WAV delta and append its PCM frames.""" - with wave.open(io.BytesIO(base64.b64decode(encoded)), "rb") as wav: - channels = wav.getnchannels() - sample_width = wav.getsampwidth() - sample_rate = wav.getframerate() - pcm_parts.append(wav.readframes(wav.getnframes())) - return channels, sample_width, sample_rate - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--base-url", default="http://localhost:8099/v1") - parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5") - parser.add_argument("--prompt", default="Say hello, then introduce vLLM in one sentence.") - parser.add_argument("--output", type=Path, default=Path("minicpmo_stream.wav")) - parser.add_argument("--text-only", action="store_true") - args = parser.parse_args() - - modalities = ["text"] if args.text_only else ["text", "audio"] - stream = OpenAI(base_url=args.base_url, api_key="EMPTY").chat.completions.create( - model=args.model, - messages=[{"role": "user", "content": args.prompt}], - modalities=modalities, - stream=True, - extra_body={"chat_template_kwargs": {"use_tts_template": not args.text_only}}, - ) - - pcm_parts: list[bytes] = [] - audio_format: tuple[int, int, int] | None = None - for chunk in stream: - for choice in chunk.choices: - content = getattr(choice.delta, "content", None) - if not content: - continue - if getattr(chunk, "modality", None) == "audio": - current_format = append_wav_chunk(content, pcm_parts) - if audio_format is not None and current_format != audio_format: - raise ValueError(f"Audio format changed: {audio_format} -> {current_format}") - audio_format = current_format - elif getattr(chunk, "modality", None) == "text": - print(content, end="", flush=True) - - print() - if pcm_parts and audio_format is not None: - channels, sample_width, sample_rate = audio_format - with wave.open(str(args.output), "wb") as wav: - wav.setnchannels(channels) - wav.setsampwidth(sample_width) - wav.setframerate(sample_rate) - wav.writeframes(b"".join(pcm_parts)) - print(f"Audio saved to {args.output} ({sample_rate} Hz)") - - -if __name__ == "__main__": - main() diff --git a/recipes/OpenBMB/MiniCPM-o-4_5.md b/recipes/OpenBMB/MiniCPM-o-4_5.md index 87f68af5d..340c89581 100644 --- a/recipes/OpenBMB/MiniCPM-o-4_5.md +++ b/recipes/OpenBMB/MiniCPM-o-4_5.md @@ -173,16 +173,18 @@ in another choice's `message.audio.data` (24 kHz mono, see Notes). With `modalities: ["text", "audio"]` you typically get two `choices` entries (one text, one audio). -**Streaming text + speech**: +**Streaming text + speech** (use `--stream`): ```bash -python examples/online_serving/minicpmo/streaming_chat_completion.py \ - --base-url http://localhost:8099/v1 \ - --output minicpmo_stream.wav +python examples/online_serving/minicpmo/openai_chat_completion_client_for_multimodal_generation.py \ + --query-type text \ + --prompt "Say hello, then introduce vLLM in one sentence." \ + --port 8099 \ + --stream ``` -The client prints text deltas as they arrive and reconstructs one valid WAV -from the independently encoded audio deltas. +The client prints text deltas as they arrive and saves streamed audio chunks +to WAV files. **Gradio demo (text + image + audio + video UI)**: diff --git a/tests/examples/online_serving/test_minicpmo_streaming.py b/tests/examples/online_serving/test_minicpmo_streaming.py deleted file mode 100644 index 90ba992eb..000000000 --- a/tests/examples/online_serving/test_minicpmo_streaming.py +++ /dev/null @@ -1,29 +0,0 @@ -import base64 -import io -import runpy -import wave -from pathlib import Path - -import pytest - -pytestmark = [pytest.mark.core_model, pytest.mark.cpu] - - -def test_append_wav_chunk_extracts_pcm_frames() -> None: - example = Path(__file__).parents[3] / "examples" / "online_serving" / "minicpmo" / "streaming_chat_completion.py" - append_wav_chunk = runpy.run_path(example)["append_wav_chunk"] - encoded_buffer = io.BytesIO() - with wave.open(encoded_buffer, "wb") as wav: - wav.setnchannels(1) - wav.setsampwidth(2) - wav.setframerate(24000) - wav.writeframes(b"\x01\x02\x03\x04") - - pcm_parts: list[bytes] = [] - audio_format = append_wav_chunk( - base64.b64encode(encoded_buffer.getvalue()).decode(), - pcm_parts, - ) - - assert audio_format == (1, 2, 24000) - assert pcm_parts == [b"\x01\x02\x03\x04"] From daec3a6a496b5c361bc70d092e34a00d72f950f5 Mon Sep 17 00:00:00 2001 From: Yash Jain Date: Thu, 30 Jul 2026 04:00:53 +0100 Subject: [PATCH 29/37] [Bugfix] Trim AR sampled logprob rows with new_token_ids on mid-step stop (#5564) Co-authored-by: Claude Fable 5 --- .../sched/test_omni_ar_scheduler_logprobs.py | 81 +++++++++++++++++++ vllm_omni/core/sched/omni_ar_scheduler.py | 8 ++ 2 files changed, 89 insertions(+) diff --git a/tests/core/sched/test_omni_ar_scheduler_logprobs.py b/tests/core/sched/test_omni_ar_scheduler_logprobs.py index 394063843..92bb30a27 100644 --- a/tests/core/sched/test_omni_ar_scheduler_logprobs.py +++ b/tests/core/sched/test_omni_ar_scheduler_logprobs.py @@ -105,6 +105,87 @@ class _RequestQueue(list): self.remove(request) +def test_mid_step_stop_trims_logprob_rows_with_token_ids() -> None: + """Spec decode can sample tokens past a stop; the trailing tokens are + trimmed by _update_request_with_output, and the emitted logprob rows must + be trimmed with them (upstream slices logprobs after the trim).""" + request = _Request("req") + request.num_computed_tokens = 8 + scheduler = SimpleNamespace( + perf_metrics=None, + connector=None, + chunk_transfer_adapter=None, + requests={request.request_id: request}, + running=[request], + waiting=_RequestQueue(), + skipped_waiting=_RequestQueue(), + structured_output_manager=SimpleNamespace(should_advance=lambda _request: False), + transfer_triggered_requests=set(), + active_kv_transfers=set(), + pending_stop_after_extraction=set(), + waiting_for_transfer_free=set(), + _new_prompt_len_snapshot={}, + finished_req_ids=set(), + finished_req_ids_dict=defaultdict(set), + kv_cache_manager=SimpleNamespace(take_events=lambda: None), + kv_event_publisher=SimpleNamespace(publish=lambda _events: None), + ) + + def update_request_trimming(req, token_ids): + # Mirror vllm Scheduler._update_request_with_output: the first sampled + # token hits EOS, so the spec-accepted tokens behind it are trimmed + # in place and the request stops. + del token_ids[1:] + req.status = RequestStatus.FINISHED_STOPPED + return token_ids, True + + def free_request(req): + scheduler.requests.pop(req.request_id, None) + scheduler.finished_req_ids.add(req.request_id) + scheduler.finished_req_ids_dict[req.client_index].add(req.request_id) + return None, None + + scheduler._update_request_with_output = update_request_trimming + scheduler._process_kv_transfer_trigger = lambda _request, _tokens: False + scheduler._handle_stopped_request = lambda _request: True + scheduler._free_request = free_request + scheduler.make_spec_decoding_stats = lambda *args, **kwargs: None + scheduler.make_stats = lambda *args, **kwargs: None + scheduler._capture_omni_connector_output = lambda _output: None + + scheduler_output = SimpleNamespace( + num_scheduled_tokens={"req": 3}, + scheduled_spec_decode_tokens={"req": [8, 9]}, + num_invalid_spec_tokens=0, + ) + model_runner_output = SimpleNamespace( + sampled_token_ids=[[7, 8, 9]], + logprobs=_logprobs( + [[7, 1], [8, 2], [9, 3]], + [[-0.7, -1.0], [-0.8, -1.1], [-0.9, -1.2]], + cumulative_rows=[0], + ), + prompt_logprobs_dict={}, + pooler_output=None, + num_nans_in_logits=None, + kv_connector_output=None, + cudagraph_stats=None, + req_id_to_index={"req": 0}, + routed_experts=None, + ) + + outputs = OmniARScheduler.update_from_output(scheduler, scheduler_output, model_runner_output) + (output,) = outputs[0].outputs + + assert output.new_token_ids == [7] + assert output.finish_reason is FinishReason.STOP + token_rows = np.asarray(output.new_logprobs.logprob_token_ids) + value_rows = np.asarray(output.new_logprobs.logprobs) + assert token_rows.shape[0] == len(output.new_token_ids) + assert value_rows.shape[0] == len(output.new_token_ids) + np.testing.assert_array_equal(token_rows[:, 0], [7]) + + def test_invalid_logprobs_finish_only_the_affected_scheduler_request() -> None: """A bad runner response must not bring down unrelated batch requests.""" bad = _Request("bad") diff --git a/vllm_omni/core/sched/omni_ar_scheduler.py b/vllm_omni/core/sched/omni_ar_scheduler.py index b043f0c0b..ecb4c4278 100644 --- a/vllm_omni/core/sched/omni_ar_scheduler.py +++ b/vllm_omni/core/sched/omni_ar_scheduler.py @@ -471,7 +471,15 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler): # Check for stop and update request status. if new_token_ids: + num_sampled_tokens = len(new_token_ids) new_token_ids, stopped = self._update_request_with_output(request, new_token_ids) + if new_logprobs is not None and len(new_token_ids) < num_sampled_tokens: + # A mid-step stop (e.g. spec-decode tokens sampled past + # EOS) trims new_token_ids after the validation slice + # above; re-slice so the emitted logprob rows stay 1:1 + # with the emitted tokens, as upstream vLLM does by + # slicing after the trim. + new_logprobs = logprobs.slice_request(req_index, len(new_token_ids)) elif request.pooling_params and pooler_output is not None: # Pooling stops as soon as there is output. request.status = RequestStatus.FINISHED_STOPPED From d50ea9fcccf6879da44f2092ff7989768c8e3666 Mon Sep 17 00:00:00 2001 From: TJian Date: Thu, 30 Jul 2026 11:27:32 +0800 Subject: [PATCH 30/37] [ROCm] [CI] Fix ROCm CI for v0.26.0 (#5528) Signed-off-by: tjtanaa --- tests/config/test_config_factory.py | 11 ++ tests/diffusion/attention/test_selector.py | 138 ++++++++++++++++++ .../models/cosmos3/test_cosmos3_pipeline.py | 5 + vllm_omni/deploy/qwen3_tts.yaml | 7 + vllm_omni/platforms/musa/platform.py | 4 +- vllm_omni/platforms/rocm/platform.py | 14 ++ 6 files changed, 178 insertions(+), 1 deletion(-) create mode 100644 tests/diffusion/attention/test_selector.py diff --git a/tests/config/test_config_factory.py b/tests/config/test_config_factory.py index 5a59dcf32..3e482f48f 100644 --- a/tests/config/test_config_factory.py +++ b/tests/config/test_config_factory.py @@ -2164,6 +2164,17 @@ class TestBaseConfigInheritance: class TestPlatformOverrides: """Test platform-specific deploy config overrides.""" + def test_qwen3_tts_rocm_disables_code2wav_outer_cudagraph(self): + deploy_path = Path(get_deploy_config_path("qwen3_tts.yaml")) + + base = load_deploy_config(deploy_path) + assert base.stages[0].enforce_eager is None + assert base.stages[1].enforce_eager is False + + rocm = _apply_platform_overrides(base, platform="rocm") + assert rocm.stages[0].enforce_eager is None + assert rocm.stages[1].enforce_eager is True + def test_npu_overrides(self): deploy_path = Path(get_deploy_config_path("qwen3_omni_moe.yaml")) if not deploy_path.exists(): diff --git a/tests/diffusion/attention/test_selector.py b/tests/diffusion/attention/test_selector.py new file mode 100644 index 000000000..79755ac2e --- /dev/null +++ b/tests/diffusion/attention/test_selector.py @@ -0,0 +1,138 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for diffusion attention backend selection.""" + +from types import SimpleNamespace + +import pytest + +import vllm_omni.diffusion.attention.selector as selector +from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec + +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + + +class _ConfiguredBackend: + @classmethod + def get_name(cls) -> str: + return "CONFIGURED" + + +class _PlatformBackend: + @classmethod + def get_name(cls) -> str: + return "PLATFORM" + + +@pytest.fixture(autouse=True) +def clear_selector_caches(): + selector._cached_get_backend_cls.cache_clear() + selector._log_backend_resolution.cache_clear() + yield + selector._cached_get_backend_cls.cache_clear() + selector._log_backend_resolution.cache_clear() + + +@pytest.mark.parametrize( + ("role", "role_category", "expected_backend"), + [ + ("ltx2.audio_to_video", "cross", "EXACT"), + ("ltx2.video_to_audio", "cross", "CATEGORY"), + ("joint", None, "DEFAULT"), + ], +) +def test_configured_selection_precedence( + monkeypatch, + role: str, + role_category: str | None, + expected_backend: str, +): + config = AttentionConfig( + default=AttentionSpec(backend="DEFAULT"), + per_role={ + "cross": AttentionSpec(backend="CATEGORY"), + "ltx2.audio_to_video": AttentionSpec(backend="EXACT"), + }, + ) + calls = [] + + def fake_get_backend(backend_name, head_size, allow_trtllm_default=True): + calls.append((backend_name, head_size, allow_trtllm_default)) + return _ConfiguredBackend + + monkeypatch.setattr(selector, "_cached_get_backend_cls", fake_get_backend) + + backend, spec = selector.get_attn_backend_for_role( + role=role, + role_category=role_category, + head_size=128, + attention_config=config, + ) + + assert backend is _ConfiguredBackend + assert spec is not None + assert spec.backend == expected_backend + assert calls == [(expected_backend, 128, True)] + + +@pytest.mark.parametrize("attention_config", [None, AttentionConfig()]) +def test_platform_default_used_without_resolved_spec(monkeypatch, attention_config): + calls = [] + + def fake_get_backend(backend_name, head_size, allow_trtllm_default=True): + calls.append((backend_name, head_size, allow_trtllm_default)) + return _PlatformBackend + + monkeypatch.setattr(selector, "_cached_get_backend_cls", fake_get_backend) + + backend, spec = selector.get_attn_backend_for_role( + role="self", + head_size=64, + attention_config=attention_config, + allow_trtllm_default=False, + ) + + assert backend is _PlatformBackend + assert spec is None + assert calls == [(None, 64, False)] + + +def test_backend_class_resolution_is_cached(monkeypatch): + platform_calls = [] + load_calls = [] + fake_platform = SimpleNamespace( + get_diffusion_attn_backend_cls=lambda **kwargs: (platform_calls.append(kwargs) or "fake.module.Backend") + ) + + monkeypatch.setattr("vllm_omni.platforms.current_omni_platform", fake_platform) + + def fake_load_backend(path): + load_calls.append(path) + return _ConfiguredBackend + + monkeypatch.setattr(selector, "_load_backend_cls", fake_load_backend) + + first = selector._cached_get_backend_cls("FLASH_ATTN", 128, False) + second = selector._cached_get_backend_cls("FLASH_ATTN", 128, False) + + assert first is second is _ConfiguredBackend + assert platform_calls == [ + { + "selected_backend": "FLASH_ATTN", + "head_size": 128, + "allow_trtllm_default": False, + } + ] + assert load_calls == ["fake.module.Backend"] + + +def test_load_backend_cls_reports_missing_module(): + with pytest.raises(ImportError, match="Failed to import module missing_attention_backend"): + selector._load_backend_cls("missing_attention_backend.Backend") + + +def test_load_backend_cls_reports_missing_class(monkeypatch): + monkeypatch.setattr(selector.importlib, "import_module", lambda _: SimpleNamespace()) + + with pytest.raises(AttributeError, match="Class MissingBackend not found in module"): + selector._load_backend_cls("fake.module.MissingBackend") diff --git a/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py b/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py index 3093e6ceb..994acd5a2 100644 --- a/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py +++ b/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py @@ -16,6 +16,7 @@ from PIL import Image from torch import nn from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch +from vllm_omni.platforms import current_omni_platform pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion] @@ -857,6 +858,10 @@ def test_flow_unipc_reuses_scheduler_and_forwards_each_request_shift(make_cosmos assert all(call["sigmas"] is None for call in scheduler.set_timesteps_calls) +@pytest.mark.skipif( + current_omni_platform.is_rocm(), + reason="Pytorch shipped with vLLM does not built with LAPACK support", +) def test_flow_unipc_reproducible_with_same_seed(make_cosmos3_pipeline) -> None: from vllm_omni.diffusion.models.schedulers.scheduling_flow_unipc_multistep import ( FlowUniPCMultistepScheduler, diff --git a/vllm_omni/deploy/qwen3_tts.yaml b/vllm_omni/deploy/qwen3_tts.yaml index 8f4c6440e..ad154bed2 100644 --- a/vllm_omni/deploy/qwen3_tts.yaml +++ b/vllm_omni/deploy/qwen3_tts.yaml @@ -97,6 +97,13 @@ stages: repetition_penalty: 1.0 platforms: + rocm: + stages: + - stage_id: 1 + # MIOpen Conv1d cannot be invoked while vLLM captures the outer + # Code2Wav model graph. Keep CUDA's graph-enabled default, but run + # Stage 1 eager on ROCm so decoder convolutions execute normally. + enforce_eager: true npu: stages: - stage_id: 0 diff --git a/vllm_omni/platforms/musa/platform.py b/vllm_omni/platforms/musa/platform.py index c71b5a70b..0774bca3a 100644 --- a/vllm_omni/platforms/musa/platform.py +++ b/vllm_omni/platforms/musa/platform.py @@ -45,6 +45,7 @@ class MUSAOmniPlatform(OmniPlatform, MUSAPlatformBase): cls, selected_backend: str | None, head_size: int, + allow_trtllm_default: bool = False, ) -> str: """Get the diffusion attention backend class path for MUSA platform. @@ -54,7 +55,8 @@ class MUSAOmniPlatform(OmniPlatform, MUSAPlatformBase): selected_backend: User-selected backend name (e.g., "FLASH_ATTN", "TORCH_SDPA"). If None, uses platform default. head_size: Attention head size. - + allow_trtllm_default: Does not support TRTLLM backend; + arg accepted for signature parity but unused. Returns: Fully qualified class path of the selected backend. """ diff --git a/vllm_omni/platforms/rocm/platform.py b/vllm_omni/platforms/rocm/platform.py index cf72baccb..6ae65ed2b 100644 --- a/vllm_omni/platforms/rocm/platform.py +++ b/vllm_omni/platforms/rocm/platform.py @@ -75,7 +75,21 @@ class RocmOmniPlatform(OmniPlatform, RocmPlatform): cls, selected_backend: str | None, head_size: int, + allow_trtllm_default: bool = False, ) -> str: + """Get the diffusion attention backend class path for ROCm platform. + + ROCm supports FLASH_ATTN via the aiter library, and SDPA as fallback. + + Args: + selected_backend: User-selected backend name (e.g., "FLASH_ATTN", + "TORCH_SDPA"). If None, uses platform default. + head_size: Attention head size. + allow_trtllm_default: Does not support TRTLLM backend; + arg accepted for signature parity but unused. + Returns: + Fully qualified class path of the selected backend. + """ from vllm._aiter_ops import is_aiter_found_and_supported # Check if aiter is available for Flash Attention support From 1d6c0816a4fbd9079878d62bbe28c1a12f146a1f Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Thu, 30 Jul 2026 11:43:08 +0800 Subject: [PATCH 31/37] [CI/Build] Fix nested group breaking nightly pipeline upload (#5572) Signed-off-by: wangyu <410167048@qq.com> --- .buildkite/cuda/test-nightly.yml | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index 377b78d1a..fd0f7cf33 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -455,24 +455,26 @@ steps: mirror_hardwares: h100_2 - group: ":card_index_dividers: Diffusion Test" + key: nightly-diffusion-test-group depends_on: upload-nightly-pipeline if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "diffusion-x2iat-test" steps: - - group: ":full_moon: Diffusion · Distributed Test" - timeout_in_minutes: 30 + - label: ":full_moon: Diffusion · Distributed Test" + timeout_in_minutes: 60 commands: - - timeout 30m pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda' --run-level "full_model" + - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda' --run-level "full_model" mirror_hardwares: l4_4 - label: ":full_moon: Diffusion · Offloader Test" - timeout_in_minutes: 40 + timeout_in_minutes: 60 commands: - - timeout 40m pytest -s -v tests/diffusion/offloader -m "full_model and cuda" + - pytest -sv tests/diffusion/offloader -m "full_model and cuda" mirror_hardwares: l4_1 - label: ":full_moon: Diffusion · Quantization Test with H100" + timeout_in_minutes: 60 commands: - - timeout 60m pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model" + - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model" mirror_hardwares: h100_1 - label: ":full_moon: Diffusion · Quantization Test with L4" @@ -482,15 +484,6 @@ steps: - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model" mirror_hardwares: l4_1 - - label: ":full_moon: Diffusion · Wan VAE Spatial Shard Correctness Test" - timeout_in_minutes: 60 - commands: - - >- - pytest -sv tests/diffusion/distributed/test_wan_spatial_shard.py - -m "full_model and diffusion and H100 and distributed_cuda" - --run-level "full_model" - mirror_hardwares: h100_2 - - label: ":bar_chart: Testcase Statistics" key: nightly-testcase-statistics timeout_in_minutes: 120 From 0c88f8a0b662ae497a6485db6f3b91b79620608c Mon Sep 17 00:00:00 2001 From: TaffyOfficial <2587297563@qq.com> Date: Thu, 30 Jul 2026 15:08:12 +0800 Subject: [PATCH 32/37] [Refactor] Normalize diffusion request extra arguments in Chat serving (#5171) Signed-off-by: TaffyOfficial <2324465096@qq.com> Co-authored-by: TaffyOfficial <2324465096@qq.com> --- .../models/bagel/test_negative_prompt.py | 78 ++++ .../comfyui/test_comfyui_integration.py | 4 +- .../test_diffusion_request_utils.py | 130 +++++++ .../test_serving_chat_sampling_params.py | 350 ++++++++++++------ .../diffusion/models/bagel/pipeline_bagel.py | 3 +- .../openai/diffusion_request_utils.py | 105 ++++++ vllm_omni/entrypoints/openai/serving_chat.py | 175 ++++++--- 7 files changed, 669 insertions(+), 176 deletions(-) create mode 100644 tests/diffusion/models/bagel/test_negative_prompt.py create mode 100644 tests/entrypoints/openai_api/test_diffusion_request_utils.py create mode 100644 vllm_omni/entrypoints/openai/diffusion_request_utils.py diff --git a/tests/diffusion/models/bagel/test_negative_prompt.py b/tests/diffusion/models/bagel/test_negative_prompt.py new file mode 100644 index 000000000..37715600c --- /dev/null +++ b/tests/diffusion/models/bagel/test_negative_prompt.py @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch +from pytest_mock import MockerFixture + +from vllm_omni.diffusion.models.bagel.pipeline_bagel import BagelPipeline +from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch +from vllm_omni.inputs.data import OmniDiffusionSamplingParams + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class _NegativePromptPreparedError(Exception): + pass + + +@pytest.mark.parametrize( + ("prompt_negative", "extra_negative", "expected"), + [ + ("prompt negative", "legacy negative", "prompt negative"), + (None, "legacy negative", "legacy negative"), + ], + ids=["prompt-owner", "legacy-extra-args"], +) +def test_forward_reads_negative_prompt_from_prompt_with_legacy_fallback( + mocker: MockerFixture, + prompt_negative: str | None, + extra_negative: str, + expected: str, +) -> None: + pipeline = object.__new__(BagelPipeline) + torch.nn.Module.__init__(pipeline) + pipeline.device = torch.device("cpu") + pipeline.od_config = mocker.Mock(dtype=torch.float32) + pipeline.tokenizer = mocker.sentinel.tokenizer + pipeline.new_token_ids = {} + pipeline.language_model = mocker.Mock(vocab_size=10) + pipeline.image_processor = None + pipeline.vae = None + + bagel = mocker.MagicMock() + bagel.max_latent_size = 32 + bagel.latent_downsample = 8 + bagel.config.llm_config.num_hidden_layers = 1 + prepared_prompts: list[str] = [] + + def prepare_prompts(*, prompts, **_): + prepared_prompts.append(prompts[0]) + if len(prepared_prompts) == 1: + return {"packed_text_ids": torch.tensor([1])}, [1], [1] + raise _NegativePromptPreparedError + + bagel.prepare_prompts.side_effect = prepare_prompts + pipeline.bagel = bagel + + prompt = { + "prompt": "positive", + "negative_prompt": prompt_negative, + } + request = DiffusionRequestBatch( + requests=[ + OmniDiffusionRequest( + prompt=prompt, + sampling_params=OmniDiffusionSamplingParams( + extra_args={"negative_prompt": extra_negative}, + ), + request_id="test", + ) + ] + ) + + with pytest.raises(_NegativePromptPreparedError): + pipeline.forward(request) + + assert prepared_prompts == ["positive", expected] diff --git a/tests/e2e/features/comfyui/test_comfyui_integration.py b/tests/e2e/features/comfyui/test_comfyui_integration.py index 2024cce8b..bfb26e897 100644 --- a/tests/e2e/features/comfyui/test_comfyui_integration.py +++ b/tests/e2e/features/comfyui/test_comfyui_integration.py @@ -35,7 +35,7 @@ from vllm.outputs import CompletionOutput, RequestOutput from vllm_omni.entrypoints.async_omni import AsyncOmni as RealAsyncOmni from vllm_omni.entrypoints.cli.serve import OmniServeCommand -from vllm_omni.inputs.data import OmniSamplingParams +from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniSamplingParams from vllm_omni.outputs import OmniRequestOutput from vllm_omni.utils.tracking_parser import TrackingArgumentParser @@ -417,7 +417,7 @@ def mock_async_omni( mock_instance.stage_configs = server_case.stage_configs mock_instance.output_modalities = _build_output_modalities(server_case.stage_configs) mock_instance.default_sampling_params_list = [ - SamplingParams() if _stage_type(stage) != "diffusion" else mocker.MagicMock() + OmniDiffusionSamplingParams() if _stage_type(stage) == "diffusion" else SamplingParams() for stage in server_case.stage_configs ] mock_instance.errored = False diff --git a/tests/entrypoints/openai_api/test_diffusion_request_utils.py b/tests/entrypoints/openai_api/test_diffusion_request_utils.py new file mode 100644 index 000000000..0b3091f24 --- /dev/null +++ b/tests/entrypoints/openai_api/test_diffusion_request_utils.py @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import pytest + +from vllm_omni.entrypoints.openai.diffusion_request_utils import ( + apply_normalized_diffusion_request_extra_args, + normalize_diffusion_request_args, +) +from vllm_omni.inputs.data import OmniDiffusionSamplingParams + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +REQUEST_SOURCES = ( + "root.declared", + "root.extra_args", + "root.extra_params", + "nested.declared", + "nested.extra_args", + "nested.extra_params", +) + + +def _source_args(source: str, value: object) -> dict[str, object]: + scope, container = source.split(".") + payload = {"solver": value} if container == "declared" else {container: {"solver": value}} + kwargs: dict[str, object] = {scope: payload} + if container == "declared": + kwargs["registered_extra_fields"] = {"solver"} + return kwargs + + +@pytest.mark.parametrize("source", REQUEST_SOURCES) +@pytest.mark.parametrize(("value", "expected"), [("ddim", "ddim"), (None, "euler")]) +def test_normalizer_routes_values_without_erasing_defaults( + source: str, + value: object, + expected: str, +) -> None: + normalized, _ = normalize_diffusion_request_args(**_source_args(source, value)) + sampling_params = OmniDiffusionSamplingParams( + extra_args={"solver": "euler", "stage_default": True}, + ) + apply_normalized_diffusion_request_extra_args(sampling_params, normalized) + + assert sampling_params.extra_args == {"solver": expected, "stage_default": True} + + +def test_non_overlapping_legacy_and_canonical_values_are_merged(mocker) -> None: + warning_once = mocker.patch("vllm_omni.entrypoints.openai.diffusion_request_utils.logger.warning_once") + + normalized, _ = normalize_diffusion_request_args( + root={"extra_args": {"cfg_text_scale": 7.0}}, + nested={"extra_params": {"sample_solver": "euler"}}, + ) + + assert normalized == {"cfg_text_scale": 7.0, "sample_solver": "euler"} + warning_once.assert_called_once() + + +@pytest.mark.parametrize( + "kwargs, expected_sources", + [ + ( + { + "root": {"seed": 1, "extra_args": {"seed": 2}}, + "serving_root_fields": {"seed"}, + }, + "request.seed, request.extra_args.seed", + ), + ( + { + "root": {"cfg_scale": 7.0}, + "nested": {"true_cfg_scale": 2.0}, + "serving_root_fields": {"true_cfg_scale"}, + "root_field_aliases": {"cfg_scale": "true_cfg_scale"}, + }, + "request.cfg_scale, request.extra_body.true_cfg_scale", + ), + ( + { + "root": { + "extra_args": {"solver": "euler"}, + "extra_params": {"solver": "ddim"}, + } + }, + "request.extra_args.solver, request.extra_params.solver", + ), + ( + { + "root": {"cfg_text_scale": 7.0}, + "nested": {"extra_args": {"cfg_text_scale": 8.0}}, + "registered_extra_fields": {"cfg_text_scale"}, + }, + "request.cfg_text_scale, request.extra_body.extra_args.cfg_text_scale", + ), + ], +) +def test_duplicate_sources_are_rejected( + kwargs: dict[str, object], + expected_sources: str, + mocker, +) -> None: + warning_once = mocker.patch("vllm_omni.entrypoints.openai.diffusion_request_utils.logger.warning_once") + + with pytest.raises(ValueError, match=expected_sources): + normalize_diffusion_request_args(**kwargs) + + warning_once.assert_not_called() + + +@pytest.mark.parametrize( + ("scope", "name"), + ( + ("root", "extra_args"), + ("root", "extra_params"), + ("nested", "extra_args"), + ("nested", "extra_params"), + ), +) +def test_public_containers_must_be_objects(scope: str, name: str) -> None: + with pytest.raises(ValueError, match="must be a JSON object"): + normalize_diffusion_request_args(**{scope: {name: ["not", "an", "object"]}}) + + +def test_container_keys_must_be_strings() -> None: + with pytest.raises(ValueError, match="must be a JSON object with string keys"): + normalize_diffusion_request_args(root={"extra_args": {1: "invalid"}}) diff --git a/tests/entrypoints/openai_api/test_serving_chat_sampling_params.py b/tests/entrypoints/openai_api/test_serving_chat_sampling_params.py index a6c7baffe..de3158d43 100644 --- a/tests/entrypoints/openai_api/test_serving_chat_sampling_params.py +++ b/tests/entrypoints/openai_api/test_serving_chat_sampling_params.py @@ -6,10 +6,12 @@ Tests that standard OpenAI API parameters (max_tokens, temperature, etc.) are correctly applied to the comprehension stage while preserving YAML defaults. """ +import asyncio from types import SimpleNamespace import pytest from pytest_mock import MockerFixture +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.sampling_params import SamplingParams pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -86,6 +88,233 @@ def serving_chat(mock_engine_client): return instance +def test_serving_boundary_normalizes_declared_root_and_nested_extras(serving_chat): + serving_chat._diffusion_extra_body_params = frozenset({"cfg_text_scale", "negative_prompt"}) + request = ChatCompletionRequest( + model="test", + messages=[], + modalities=["image"], + num_inference_steps=7, + size="768x512", + negative_prompt="avoid blur", + lora={"name": "adapter"}, + cfg_text_scale=7.0, + extra_body={"extra_args": {"sample_solver": "euler"}}, + ) + + normalized_extra_args, diffusion_request_args = serving_chat._normalize_diffusion_request_args(request) + + assert normalized_extra_args == { + "cfg_text_scale": 7.0, + "sample_solver": "euler", + } + assert diffusion_request_args["cfg_text_scale"] == 7.0 + assert diffusion_request_args["num_inference_steps"] == 7 + assert diffusion_request_args["size"] == "768x512" + assert diffusion_request_args["negative_prompt"] == "avoid blur" + assert diffusion_request_args["lora"] == {"name": "adapter"} + assert diffusion_request_args["modalities"] == ["image"] + + +def test_unknown_root_extra_does_not_claim_canonical_extra(serving_chat): + serving_chat._diffusion_extra_body_params = frozenset() + request = ChatCompletionRequest( + model="test", + messages=[], + pipeline_option="ignored-root-value", + extra_args={"pipeline_option": "canonical-value"}, + ) + + normalized_extra_args, diffusion_request_args = serving_chat._normalize_diffusion_request_args(request) + + assert normalized_extra_args == { + "pipeline_option": "canonical-value", + } + assert "pipeline_option" not in diffusion_request_args + + +def test_unregistered_cfg_scale_aliases_common_true_cfg_scale(serving_chat): + serving_chat._diffusion_extra_body_params = frozenset() + request = ChatCompletionRequest( + model="test", + messages=[], + cfg_scale=7.0, + ) + + normalized_extra_args, diffusion_request_args = serving_chat._normalize_diffusion_request_args(request) + + assert normalized_extra_args == {} + assert diffusion_request_args == {"true_cfg_scale": 7.0} + + +@pytest.mark.parametrize( + ("registered", "request_kwargs"), + [ + ({"cfg_scale"}, {"cfg_scale": 7.0, "extra_args": {"cfg_scale": 8.0}}), + (set(), {"cfg_scale": 7.0, "true_cfg_scale": 2.0}), + ], + ids=["registered-model-extra", "unregistered-common-alias"], +) +def test_cfg_scale_owner_conflicts_are_rejected(serving_chat, registered, request_kwargs): + serving_chat._diffusion_extra_body_params = frozenset(registered) + request = ChatCompletionRequest(model="test", messages=[], **request_kwargs) + + with pytest.raises(ValueError, match="provided more than once"): + serving_chat._normalize_diffusion_request_args(request) + + +@pytest.mark.parametrize("diffusion_mode", [True, False], ids=["pure", "mixed"]) +def test_duplicate_extras_return_the_same_bad_request_before_dispatch( + serving_chat, + mocker: MockerFixture, + diffusion_mode: bool, +): + serving_chat._diffusion_mode = diffusion_mode + serving_chat._diffusion_extra_body_params = frozenset({"sample_solver"}) + serving_chat.engine_client.stage_configs = [SimpleNamespace(stage_type="diffusion")] + check_model = mocker.patch.object(serving_chat, "_check_model", new=mocker.AsyncMock()) + pure_dispatch = mocker.patch.object( + serving_chat, + "_create_diffusion_chat_completion", + new=mocker.AsyncMock(), + ) + request = ChatCompletionRequest( + model="test", + messages=[], + sample_solver="euler", + extra_args={"sample_solver": "ddim"}, + ) + + response = asyncio.run(serving_chat._create_chat_completion(request)) + + assert response.error.code == 400 + assert response.error.message == ( + 'Diffusion request parameters were provided more than once: "sample_solver": ' + "request.sample_solver, request.extra_args.sample_solver." + ) + check_model.assert_not_awaited() + pure_dispatch.assert_not_awaited() + + +def test_pure_consumer_preserves_defaults_and_separate_cfg_owners(serving_chat, mocker: MockerFixture): + from vllm_omni.inputs.data import OmniDiffusionSamplingParams + + captured: dict[str, object] = {} + + async def generate(**kwargs): + captured.update(kwargs) + if False: + yield None + + serving_chat._diffusion_model_name = "test" + serving_chat._diffusion_engine = SimpleNamespace( + stage_configs=[SimpleNamespace(stage_type="diffusion")], + default_sampling_params_list=[ + OmniDiffusionSamplingParams(extra_args={"solver": "euler", "stage_default": True}), + ], + generate=generate, + ) + serving_chat._diffusion_mode = True + serving_chat._diffusion_extra_body_params = frozenset({"cfg_scale"}) + serving_chat._extract_diffusion_prompt_and_media = mocker.Mock(return_value=("prompt", [], [], [])) + request = ChatCompletionRequest( + model="test", + messages=[], + modalities=["image"], + num_inference_steps=7, + size="768x512", + negative_prompt="avoid blur", + cfg_scale=7.0, + true_cfg_scale=2.0, + extra_body={"extra_args": {"solver": "ddim"}}, + ) + + response = asyncio.run(serving_chat._create_chat_completion(request)) + + (sampling_params,) = captured["sampling_params_list"] + assert sampling_params.extra_args == { + "cfg_scale": 7.0, + "solver": "ddim", + "stage_default": True, + } + assert sampling_params.true_cfg_scale == 2.0 + assert sampling_params.num_inference_steps == 7 + assert (sampling_params.height, sampling_params.width) == (512, 768) + assert captured["prompt"]["negative_prompt"] == "avoid blur" + assert response.error.message == "No output generated from AsyncOmni" + + +def test_mixed_consumer_keeps_root_common_args_with_nested_extras(serving_chat, mocker: MockerFixture): + from vllm_omni.inputs.data import OmniDiffusionSamplingParams + + serving_chat.engine_client.stage_configs = [ + SimpleNamespace(stage_type="llm", is_comprehension=True), + SimpleNamespace(stage_type="diffusion", is_comprehension=False), + ] + serving_chat.engine_client.default_sampling_params_list = [ + SamplingParams(), + OmniDiffusionSamplingParams(), + ] + serving_chat.engine_client.output_modalities = ["image"] + serving_chat.engine_client.errored = False + + captured: dict[str, object] = {} + + async def results(): + if False: + yield None + + def generate(**kwargs): + captured.update(kwargs) + return results() + + serving_chat.engine_client.generate = generate + serving_chat.__dict__.update( + _diffusion_mode=False, + _diffusion_extra_body_params=frozenset(), + models=SimpleNamespace(model_name=lambda _: "test"), + renderer=SimpleNamespace(get_tokenizer=lambda: object()), + online_renderer=SimpleNamespace(validate_chat_template=lambda **_: None), + parser_cls=None, + use_harmony=False, + enable_auto_tools=False, + exclude_tools_when_tool_choice_none=False, + trust_request_chat_template=True, + chat_template=None, + chat_template_content_format="auto", + ) + mocker.patch.multiple( + serving_chat, + _check_model=mocker.AsyncMock(return_value=None), + _maybe_get_adapters=mocker.Mock(return_value=None), + _effective_chat_template_kwargs=mocker.Mock(return_value={}), + _preprocess_chat=mocker.AsyncMock(return_value=([], [{"prompt": "raw"}])), + _base_request_id=mocker.Mock(return_value="test"), + _extract_diffusion_prompt_and_images_from_messages=mocker.Mock(return_value=("prompt", [])), + _log_inputs=mocker.Mock(), + chat_completion_full_generator=mocker.AsyncMock(return_value="done"), + ) + request = ChatCompletionRequest( + model="test", + messages=[{"role": "user", "content": "prompt"}], + modalities=["image"], + num_inference_steps=7, + size="768x512", + negative_prompt="avoid blur", + extra_body={"extra_args": {"solver": "euler"}}, + ) + + assert asyncio.run(serving_chat._create_chat_completion(request)) == "done" + + sampling_params_list = captured["sampling_params_list"] + assert sampling_params_list[0].extra_args == {"target_h": 512, "target_w": 768} + diffusion_params = sampling_params_list[1] + assert diffusion_params.num_inference_steps == 7 + assert (diffusion_params.height, diffusion_params.width) == (512, 768) + assert diffusion_params.extra_args == {"solver": "euler"} + assert captured["prompt"]["negative_prompt"] == "avoid blur" + + @pytest.fixture def mock_request(mocker: MockerFixture): """Create a mock request with all OpenAI sampling params set to None.""" @@ -616,124 +845,3 @@ class TestResolveHeightWidth: h, w = OmniOpenAIServingChat._resolve_height_width_from_extra_body({"size": "invalid"}) assert h is None assert w is None - - -# ============================================================================= -# Tests for _apply_request_overrides with GLM-Image (target_h/w injection) -# ============================================================================= - - -class TestApplyRequestOverridesGLMImage: - """Test target_h/w injection for GLM-Image AR stage. - - max_tokens is NOT computed dynamically — it comes from the deploy YAML - default (e.g. 4353). _apply_request_overrides only injects target_h/w - into extra_args so the model can build M-RoPE position grids. - """ - - @pytest.fixture - def glm_serving_chat(self, mock_engine_client, mocker: MockerFixture): - from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat - - instance = object.__new__(OmniOpenAIServingChat) - instance.engine_client = mock_engine_client - instance._extract_diffusion_prompt_and_images_from_messages = mocker.MagicMock(return_value=("a cat", [])) - return instance - - @pytest.fixture - def glm_request(self, mocker: MockerFixture): - req = mocker.MagicMock() - req.temperature = None - req.top_p = None - req.top_k = None - req.max_tokens = None - req.min_tokens = None - req.seed = None - req.ignore_eos = None - req.stop = None - req.stop_token_ids = None - req.frequency_penalty = None - req.presence_penalty = None - req.extra_body = {"height": 1024, "width": 1024} - req.model_fields_set = set() - return req - - def test_t2i_injects_target_h_w(self, glm_serving_chat, glm_request, default_comprehension_params): - """t2i mode: target_h/w injected into extra_args, max_tokens unchanged.""" - result = glm_serving_chat._apply_request_overrides(default_comprehension_params, glm_request) - assert result.extra_args["target_h"] == 1024 - assert result.extra_args["target_w"] == 1024 - # max_tokens stays at YAML default (not dynamically computed) - assert result.max_tokens == 4353 - - def test_i2i_injects_target_h_w( - self, glm_serving_chat, glm_request, default_comprehension_params, mocker: MockerFixture - ): - """i2i mode: target_h/w injected, max_tokens unchanged.""" - glm_serving_chat._extract_diffusion_prompt_and_images_from_messages = mocker.MagicMock( - return_value=("edit this", ["fake_image"]) - ) - result = glm_serving_chat._apply_request_overrides(default_comprehension_params, glm_request) - assert result.extra_args["target_h"] == 1024 - assert result.extra_args["target_w"] == 1024 - # max_tokens stays at YAML default regardless of t2i/i2i - assert result.max_tokens == 4353 - - def test_user_max_tokens_preserved(self, glm_serving_chat, glm_request, default_comprehension_params): - """User-provided max_tokens is respected (not overridden by dynamic computation).""" - glm_request.max_tokens = 500 - glm_request.model_fields_set = {"max_tokens"} - - result = glm_serving_chat._apply_request_overrides(default_comprehension_params, glm_request) - assert result.max_tokens == 500 - assert result.extra_args["target_h"] == 1024 - assert result.extra_args["target_w"] == 1024 - - def test_no_height_width_preserves_default( - self, glm_serving_chat, mocker: MockerFixture, default_comprehension_params - ): - """When no height/width in extra_body, keep YAML default max_tokens, no target_h/w.""" - req = mocker.MagicMock() - req.temperature = None - req.top_p = None - req.top_k = None - req.max_tokens = None - req.min_tokens = None - req.seed = None - req.ignore_eos = None - req.stop = None - req.stop_token_ids = None - req.frequency_penalty = None - req.presence_penalty = None - req.extra_body = {} - req.model_fields_set = set() - - result = glm_serving_chat._apply_request_overrides(default_comprehension_params, req) - assert result.max_tokens == 4353 # YAML default - # No target_h/w injected when dimensions not provided - assert not result.extra_args or "target_h" not in (result.extra_args or {}) - - def test_size_string_parsed_for_glm_image( - self, glm_serving_chat, mocker: MockerFixture, default_comprehension_params - ): - """'size' in extra_body is parsed as fallback for height/width.""" - req = mocker.MagicMock() - req.temperature = None - req.top_p = None - req.top_k = None - req.max_tokens = None - req.min_tokens = None - req.seed = None - req.ignore_eos = None - req.stop = None - req.stop_token_ids = None - req.frequency_penalty = None - req.presence_penalty = None - req.extra_body = {"size": "512x512"} - req.model_fields_set = set() - - result = glm_serving_chat._apply_request_overrides(default_comprehension_params, req) - assert result.extra_args["target_h"] == 512 - assert result.extra_args["target_w"] == 512 - # max_tokens stays at YAML default (not dynamically computed) - assert result.max_tokens == 4353 diff --git a/vllm_omni/diffusion/models/bagel/pipeline_bagel.py b/vllm_omni/diffusion/models/bagel/pipeline_bagel.py index 9786e2b47..6ca306c60 100644 --- a/vllm_omni/diffusion/models/bagel/pipeline_bagel.py +++ b/vllm_omni/diffusion/models/bagel/pipeline_bagel.py @@ -610,7 +610,8 @@ class BagelPipeline(nn.Module, SupportsComponentDiscovery, DiffusionPipelineProf # cfg_text_context: update with negative prompt (no text condition). # When empty, keep cfg_text_context as-is (kv_lens=0) to match # original BAGEL. - neg_prompt = extra_args.get("negative_prompt", "") + prompt_negative = first_prompt.get("negative_prompt") if isinstance(first_prompt, dict) else None + neg_prompt = prompt_negative if prompt_negative is not None else extra_args.get("negative_prompt", "") if neg_prompt: neg_input, neg_newlens, neg_rope = self.bagel.prepare_prompts( curr_kvlens=cfg_text_context["kv_lens"], diff --git a/vllm_omni/entrypoints/openai/diffusion_request_utils.py b/vllm_omni/entrypoints/openai/diffusion_request_utils.py new file mode 100644 index 000000000..84fec8a3a --- /dev/null +++ b/vllm_omni/entrypoints/openai/diffusion_request_utils.py @@ -0,0 +1,105 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +from collections.abc import Collection, Mapping + +from vllm.logger import init_logger + +from vllm_omni.inputs.data import OmniDiffusionSamplingParams + +logger = init_logger(__name__) + + +def _request_mapping(name: str, value: object | None) -> dict[str, object]: + if value is None: + return {} + if not isinstance(value, Mapping): + raise ValueError(f"{name} must be a JSON object.") + if any(not isinstance(key, str) for key in value): + raise ValueError(f"{name} must be a JSON object with string keys.") + return dict(value) + + +def normalize_diffusion_request_args( + *, + root: object | None = None, + nested: object | None = None, + explicit_root_args: object | None = None, + serving_root_fields: Collection[str] = (), + registered_extra_fields: Collection[str] = (), + root_field_aliases: Mapping[str, str] | None = None, +) -> tuple[dict[str, object], dict[str, object]]: + """Validate request sources and project diffusion consumer arguments.""" + root_args = _request_mapping("request", root) + root_args.update(_request_mapping("explicit_root_args", explicit_root_args)) + nested_args = _request_mapping("request.extra_body", nested) + + registered = set(registered_extra_fields) + aliases = {alias: canonical for alias, canonical in (root_field_aliases or {}).items() if alias not in registered} + serving_fields = set(serving_root_fields) | aliases.keys() + declared_fields = registered - serving_fields + consumer_fields = serving_fields | declared_fields + + root_declared = {key: root_args[key] for key in declared_fields if key in root_args} + nested_declared = {key: nested_args[key] for key in declared_fields if key in nested_args} + canonical = _request_mapping("extra_args", root_args.get("extra_args")) + legacy = _request_mapping("extra_params", root_args.get("extra_params")) + nested_canonical = _request_mapping("extra_body.extra_args", nested_args.get("extra_args")) + nested_legacy = _request_mapping("extra_body.extra_params", nested_args.get("extra_params")) + + sources_by_key: dict[str, list[str]] = {} + sources = ( + ((root_args.keys() & serving_fields) | root_declared.keys(), "request"), + ((nested_args.keys() & serving_fields) | nested_declared.keys(), "request.extra_body"), + (canonical, "request.extra_args"), + (legacy, "request.extra_params"), + (nested_canonical, "request.extra_body.extra_args"), + (nested_legacy, "request.extra_body.extra_params"), + ) + for keys, prefix in sources: + for key in sorted(keys): + sources_by_key.setdefault(aliases.get(key, key), []).append(f"{prefix}.{key}") + + conflicts = {key: list(dict.fromkeys(paths)) for key, paths in sources_by_key.items() if len(set(paths)) > 1} + if conflicts: + details = "; ".join(f'"{key}": {", ".join(conflicts[key])}' for key in sorted(conflicts)) + raise ValueError(f"Diffusion request parameters were provided more than once: {details}.") + + if root_args.get("extra_params") is not None or nested_args.get("extra_params") is not None: + logger.warning_once( + "extra_params is deprecated; use extra_args for model-specific diffusion request parameters." + ) + + # Duplicate request keys were rejected above, so ordering is not precedence. + normalized_extra_args = { + **nested_declared, + **root_declared, + **legacy, + **nested_legacy, + **canonical, + **nested_canonical, + } + normalized_extra_args = {key: value for key, value in normalized_extra_args.items() if value is not None} + + request_args = { + **{key: nested_args[key] for key in consumer_fields if key in nested_args}, + **{key: root_args[key] for key in consumer_fields if key in root_args}, + } + for alias, canonical_name in aliases.items(): + if alias in request_args: + request_args[canonical_name] = request_args.pop(alias) + return normalized_extra_args, request_args + + +def apply_normalized_diffusion_request_extra_args( + sampling_params: OmniDiffusionSamplingParams, + normalized_extra_args: Mapping[str, object], +) -> None: + """Overlay request extras without discarding stage defaults.""" + if normalized_extra_args: + sampling_params.extra_args = { + **(sampling_params.extra_args or {}), + **normalized_extra_args, + } diff --git a/vllm_omni/entrypoints/openai/serving_chat.py b/vllm_omni/entrypoints/openai/serving_chat.py index a0bce7894..7c807cfc9 100755 --- a/vllm_omni/entrypoints/openai/serving_chat.py +++ b/vllm_omni/entrypoints/openai/serving_chat.py @@ -25,6 +25,10 @@ from vllm.entrypoints.chat_utils import ( from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args from vllm_omni.entrypoints.async_omni import AsyncOmni +from vllm_omni.entrypoints.openai.diffusion_request_utils import ( + apply_normalized_diffusion_request_extra_args, + normalize_diffusion_request_args, +) from vllm_omni.entrypoints.openai.protocol.chat_completion import OmniChatCompletionResponse from vllm_omni.entrypoints.utils import coerce_param_message_types from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt @@ -33,7 +37,10 @@ from vllm_omni.metrics.modality import ( observe_audio_first_packet, observe_audio_streaming_finalize, ) -from vllm_omni.model_extras import get_extra_body_params, get_extra_output_params +from vllm_omni.model_extras import ( + get_extra_body_params, + get_extra_output_params, +) try: import soundfile @@ -95,7 +102,10 @@ from vllm.utils.collection_utils import as_list from vllm.v1.engine.exceptions import EngineDeadError from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin -from vllm_omni.entrypoints.openai.image_api_utils import encode_image_base64_with_compression, validate_layered_layers +from vllm_omni.entrypoints.openai.image_api_utils import ( + encode_image_base64_with_compression, + validate_layered_layers, +) from vllm_omni.entrypoints.openai.protocol import OmniChatCompletionStreamResponse from vllm_omni.entrypoints.openai.protocol.audio import ( DEFAULT_AUDIO_FORMAT, @@ -152,6 +162,33 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): _supported_speakers: set[str] | None = None _diffusion_extra_body_params: frozenset[str] | None = None _diffusion_extra_output_params: frozenset[str] | None = None + # Common fields stay at the request root, but still participate in + # conflicts with model-specific compatibility forms. + _diffusion_common_root_fields = frozenset( + { + "height", + "width", + "num_outputs_per_prompt", + "seed", + "num_inference_steps", + "guidance_scale", + "true_cfg_scale", + "num_frames", + "guidance_scale_2", + "layers", + "resolution", + } + ) + _diffusion_existing_control_fields = frozenset( + { + "size", + "negative_prompt", + "lora", + "modalities", + } + ) + _diffusion_root_field_aliases = {"cfg_scale": "true_cfg_scale"} + _diffusion_serving_root_fields = _diffusion_common_root_fields | _diffusion_existing_control_fields # Harmony flag (always False for vllm-omni models) use_harmony: bool = False @@ -291,6 +328,30 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): self._diffusion_extra_body_params = params return params + def _normalize_diffusion_request_args( + self, + request: ChatCompletionRequest, + ) -> tuple[dict[str, object], dict[str, Any]]: + """Normalize extras and preserve validated common root fields.""" + root = dict(request.model_extra or {}) + nested = root.pop("extra_body", None) + if nested is None: + nested = getattr(request, "extra_body", None) + explicit = getattr(request, "model_fields_set", None) + if explicit is None: + explicit = getattr(request, "__fields_set__", ()) + explicit_root_args = { + key: getattr(request, key) for key in explicit if key != "extra_body" and hasattr(request, key) + } + return normalize_diffusion_request_args( + root=root, + nested=nested, + explicit_root_args=explicit_root_args, + serving_root_fields=self._diffusion_serving_root_fields, + registered_extra_fields=self._get_diffusion_extra_body_params(), + root_field_aliases=self._diffusion_root_field_aliases, + ) + def _get_diffusion_extra_output_params( self, output: object, @@ -386,9 +447,26 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): request: ChatCompletionRequest, raw_request: Request | None = None, ) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse: + stage_configs = getattr(self.engine_client, "stage_configs", ()) or () + serves_diffusion = self._diffusion_mode or any( + get_stage_type(stage_config) == "diffusion" for stage_config in stage_configs + ) + normalized_extra_args: dict[str, object] = {} + diffusion_request_args: dict[str, Any] = {} + if serves_diffusion: + try: + normalized_extra_args, diffusion_request_args = self._normalize_diffusion_request_args(request) + except ValueError as exc: + return self._create_error_response(str(exc), status_code=400) + # Handle diffusion mode if self._diffusion_mode: - return await self._create_diffusion_chat_completion(request, raw_request) + return await self._create_diffusion_chat_completion( + request, + normalized_extra_args, + diffusion_request_args, + raw_request, + ) request_timestamp = time.time() if raw_request is not None: @@ -530,7 +608,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): return audio_format_check num_inference_steps = None - extra_body: dict[str, Any] = {} + extra_body = diffusion_request_args # Omni multistage image generation: Stage-0 (AR) should receive a clean # text prompt (and optional conditioning image/size) so the model's own # processor can construct the correct inputs. @@ -544,12 +622,6 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): if not extracted_prompt: return self.create_error_response("No text prompt found in messages") - # [NOTE] When sending request via openai client Python library, - # `extra_body` is flattented and merged into the payload's root. - # These extra fields are accessible via `model_extra` property (from Pydantic base class). - # When sending raw request with curl, no flattening happens. Directly read the `extra_body` dict. - extra_body = getattr(request, "extra_body", None) or request.model_extra or {} - height, width = self._resolve_height_width_from_extra_body(extra_body) num_inference_steps = extra_body.get("num_inference_steps") @@ -668,15 +740,32 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): output_modalities, ) + comprehension_idx = ( + self._get_comprehension_stage_index() + if _image_gen_height is not None and _image_gen_width is not None + else None + ) # Apply user-specified overrides to diffusion stage(s) for image generation for idx, sp in enumerate(sampling_params_list): + if idx == comprehension_idx: + extra_args = dict(getattr(sp, "extra_args", {}) or {}) + extra_args["target_h"] = int(_image_gen_height) + extra_args["target_w"] = int(_image_gen_width) + sp.extra_args = extra_args if hasattr(sp, "height") and _image_gen_height is not None: sp.height = _image_gen_height if hasattr(sp, "width") and _image_gen_width is not None: sp.width = _image_gen_width if hasattr(sp, "num_inference_steps") and num_inference_steps is not None: sp.num_inference_steps = num_inference_steps - apply_declared_extra_args(sp, self._get_diffusion_extra_body_params(), extra_body) + if isinstance(sp, OmniDiffusionSamplingParams): + apply_normalized_diffusion_request_extra_args(sp, normalized_extra_args) + else: + apply_declared_extra_args( + sp, + self._get_diffusion_extra_body_params(), + extra_body, + ) self._log_inputs( request_id, @@ -1166,8 +1255,6 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): Starts with YAML defaults and only overrides fields that the user explicitly provided (non-None values) in the request. - For models needing spatial metadata (e.g. GLM-Image), target_h/w is - injected into extra_args so the runner can build M-RoPE position grids. max_tokens is NOT computed dynamically — it uses the deploy YAML default. Args: @@ -1193,19 +1280,6 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): if (value is not None and not isinstance(value, list)) or (isinstance(value, list) and len(value) > 0): setattr(params, field_name, value) - # For GLM-Image: compute max_tokens from height/width with mode-aware - # budgeting (t2i vs i2i). - extra_body = getattr(request, "extra_body", {}) or {} - height, width = self._resolve_height_width_from_extra_body(extra_body) - - if height is not None and width is not None: - # Keep target size in stage-0 sampling params so runner/model can - # build deterministic M-RoPE grids for t2i (no MM features). - extra_args = dict(getattr(params, "extra_args", {}) or {}) - extra_args["target_h"] = int(height) - extra_args["target_w"] = int(width) - params.extra_args = extra_args - return params @staticmethod @@ -3366,12 +3440,16 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): async def _create_diffusion_chat_completion( self, request: ChatCompletionRequest, + normalized_extra_args: dict[str, object], + diffusion_request_args: dict[str, Any], raw_request: Request | None = None, ) -> ChatCompletionResponse | ErrorResponse: """Generate images via chat completion interface for diffusion models. Args: request: Chat completion request + normalized_extra_args: Validated model-specific request arguments + diffusion_request_args: Compatibility arguments plus validated common root fields raw_request: Raw FastAPI request object Returns: @@ -3396,15 +3474,8 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): messages ) - # Extract generation parameters from extra_body (preferred) # Reference: text_to_image.py and text_to_video.py for supported parameters - # [NOTE] When sending request via openai client Python library, - # `extra_body` is flattented and merged into the payload's root. - # These extra fields are accessible via `model_extra` property (from Pydantic base class). - # When sending raw request with curl, no flattening happens. Directly read the `extra_body` dict. - extra_body = getattr(request, "extra_body", None) - if not extra_body: - extra_body = request.model_extra or {} + extra_body = diffusion_request_args # Parse size if provided (supports "1024x1024" format) height, width = self._resolve_height_width_from_extra_body(extra_body) @@ -3415,7 +3486,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): # not provide a value. num_inference_steps = extra_body.get("num_inference_steps") guidance_scale = extra_body.get("guidance_scale") - true_cfg_scale = extra_body.get("true_cfg_scale") or extra_body.get("cfg_scale") + true_cfg_scale = extra_body.get("true_cfg_scale") seed = extra_body.get("seed") if seed is None: seed = getattr(request, "seed", None) @@ -3430,18 +3501,17 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): # Qwen-Image-Layered parameters layers = extra_body.get("layers") resolution = extra_body.get("resolution") - try: layers = validate_layered_layers(layers) - except ValueError as e: - return self._create_error_response(str(e), status_code=400) + except ValueError as exc: + return self._create_error_response(str(exc), status_code=400) logger.info( "Diffusion chat request %s: prompt=%r, ref_images=%d, params=%s", request_id, prompt[:50] + "..." if len(prompt) > 50 else prompt, len(reference_images), - {k: v for k, v in extra_body.items() if v is not None}, + {key: value for key, value in extra_body.items() if value is not None}, ) # Decode reference images if provided @@ -3473,7 +3543,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): gen_params.guidance_scale = guidance_scale if true_cfg_scale is not None: gen_params.true_cfg_scale = true_cfg_scale - apply_declared_extra_args(gen_params, self._get_diffusion_extra_body_params(), extra_body) + apply_normalized_diffusion_request_extra_args(gen_params, normalized_extra_args) if num_frames is not None: gen_params.num_frames = num_frames if guidance_scale_2 is not None: @@ -3483,14 +3553,6 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): if resolution is not None: gen_params.resolution = resolution - # Pipeline-agnostic escape hatch (mirrors ``extra_params`` on the /v1/videos - # endpoint in ``serving_video.py``): a single reserved ``extra_args`` dict in - # ``extra_body`` flows straight into ``gen_params.extra_args``, with no keys - # hardcoded here. - extra_args_body = extra_body.get("extra_args") - if isinstance(extra_args_body, dict): - gen_params.extra_args.update(extra_args_body) - # Parse per-request LoRA. if lora_body and isinstance(lora_body, dict): try: @@ -3499,8 +3561,8 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): gen_params.lora_request = lora_req if lora_scale is not None: gen_params.lora_scale = lora_scale - except Exception as e: # pragma: no cover - safeguard - logger.warning("Failed to parse LoRA request: %s", e) + except Exception as exc: # pragma: no cover - safeguard + logger.warning("Failed to parse LoRA request: %s", exc) # Route text modality for single-stage diffusion (img2text / text2text) requested_modalities = extra_body.get("modalities") or [] @@ -3539,13 +3601,22 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): # Generate image or audio (e.g. AudioX) via AsyncOmni diffusion_engine = cast(AsyncOmni, self._diffusion_engine) stage_configs = list(getattr(diffusion_engine, "stage_configs", []) or []) + default_sampling_params_list = get_default_sampling_params_list(diffusion_engine) sampling_params_list = build_stage_sampling_params_list( stage_configs, - get_default_sampling_params_list(diffusion_engine), + default_sampling_params_list, diffusion_params=gen_params, replace_diffusion_params=True, ) - + for index, (stage_config, sampling_params) in enumerate(zip(stage_configs, sampling_params_list)): + if get_stage_type(stage_config) != "diffusion" or index >= len(default_sampling_params_list): + continue + default_extra_args = getattr(default_sampling_params_list[index], "extra_args", None) + if default_extra_args: + sampling_params.extra_args = { + **default_extra_args, + **(getattr(sampling_params, "extra_args", None) or {}), + } if not sampling_params_list: sampling_params_list = [gen_params] From 04b408ffd2251e7e24db250b5174209a6838735c Mon Sep 17 00:00:00 2001 From: SYLAR <125541396+lishunyang12@users.noreply.github.com> Date: Thu, 30 Jul 2026 16:41:30 +0800 Subject: [PATCH 33/37] [Bugfix] Add Cosmos3-Edge preset (fix wrong Nano/Super defaults in T2V/I2V examples) (#5596) Signed-off-by: lishunyang12 Co-authored-by: lishunyang12 --- .../image_to_video/image_to_video.py | 15 ++++++++++++++- .../text_to_video/text_to_video.py | 17 +++++++++++++++++ recipes/cosmos3/Cosmos3-Edge.md | 8 +++++++- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/examples/offline_inference/image_to_video/image_to_video.py b/examples/offline_inference/image_to_video/image_to_video.py index cbda2c3f0..b2f27bb5b 100644 --- a/examples/offline_inference/image_to_video/image_to_video.py +++ b/examples/offline_inference/image_to_video/image_to_video.py @@ -334,6 +334,7 @@ def main(): is_ltx23 = "ltx23" in model_class_name_lower or "ltx-2.3" in model_name is_ltx2 = is_ltx2_distilled or is_ltx23 or "ltx2" in model_class_name_lower or "ltx-2" in model_name is_cosmos = "cosmos" in model_name or (model_class_name is not None and "cosmos" in model_class_name.lower()) + is_cosmos_edge = is_cosmos and ("edge" in model_name or "edge" in model_class_name_lower) image = PIL.Image.open(args.image).convert("RGB") if args.image else None last_image = PIL.Image.open(args.last_image).convert("RGB") if args.last_image else None @@ -347,7 +348,19 @@ def main(): # Per-model generation defaults, applied only when the matching flag is omitted. # Cosmos3 would otherwise silently inherit the Wan2.2 defaults (wrong size/steps/shift). - if is_cosmos: + if is_cosmos_edge: + # Cosmos3-Edge native defaults: 480x832, gs 5.0, flow_shift 3.0 — NOT the Nano/Super + # 720p / gs 6.0 / flow_shift 10.0 below, which produce degenerate output on Edge. + d_fps, d_guidance, d_num_frames, d_steps, d_flow_shift, d_max_area, d_mod = ( + 24, + 5.0, + 189, + 35, + 3.0, + 480 * 832, + 16, + ) + elif is_cosmos: d_fps, d_guidance, d_num_frames, d_steps, d_flow_shift, d_max_area, d_mod = ( 24, 6.0, diff --git a/examples/offline_inference/text_to_video/text_to_video.py b/examples/offline_inference/text_to_video/text_to_video.py index b0c8a2509..b21ec8f48 100644 --- a/examples/offline_inference/text_to_video/text_to_video.py +++ b/examples/offline_inference/text_to_video/text_to_video.py @@ -56,6 +56,19 @@ _MODEL_PRESETS = { "flow_shift": 10.0, "output": "cosmos3_t2v_output.mp4", }, + # Cosmos3-Edge is a compact checkpoint with its own native resolution / guidance / + # flow_shift. It must NOT inherit the Nano/Super "cosmos" numbers above (720p / gs 6.0 / + # flow_shift 10.0), which produce degenerate output on Edge. + "cosmos3_edge": { + "height": 480, + "width": 832, + "num_frames": 189, + "num_inference_steps": 35, + "guidance_scale": 5.0, + "fps": 24, + "flow_shift": 3.0, + "output": "cosmos3_edge_t2v_output.mp4", + }, "helios": { "height": 384, "width": 640, @@ -103,6 +116,10 @@ def _detect_preset(model: str, model_class_name: str | None = None) -> dict: return _MODEL_PRESETS["ltx2"] if "vace" in model_lower: return _MODEL_PRESETS["vace"] + # Edge must be matched before the generic cosmos branch (its "cosmos" substring would + # otherwise pick up the Nano/Super 720p / gs 6.0 / flow_shift 10.0 preset). + if ("cosmos" in model_lower or "cosmos" in class_lower) and ("edge" in model_lower or "edge" in class_lower): + return _MODEL_PRESETS["cosmos3_edge"] if "cosmos" in model_lower: return _MODEL_PRESETS["cosmos"] if "hunyuan" in model_lower: diff --git a/recipes/cosmos3/Cosmos3-Edge.md b/recipes/cosmos3/Cosmos3-Edge.md index dc783236e..eeb5c3083 100644 --- a/recipes/cosmos3/Cosmos3-Edge.md +++ b/recipes/cosmos3/Cosmos3-Edge.md @@ -17,7 +17,9 @@ The small Cosmos3 checkpoint, for constrained deployments. It runs on the shared - **Transformer**: `Cosmos3EdgeVFMTransformer` (Nemotron-based), auto-selected from `backbone_type: cosmos3_edge_nemotron_dense` — nothing to pass. -- **Defaults** differ; omit `--height` / `--width` to get them: +- **Defaults** differ. The offline examples auto-detect Edge (from `edge` in the model id / + transformer class) and apply the Edge column below, so you can omit `--height` / `--width` / + `--guidance-scale` / `--flow-shift` and still get native Edge output: | | Edge | Nano / Super | |---|---|---| @@ -26,6 +28,10 @@ The small Cosmos3 checkpoint, for constrained deployments. It runs on the shared | Video `guidance_scale` | 5.0 | 6.0 | | Video `flow_shift` | 3.0 | 10.0 | +> **Important:** these auto-defaults live in the example CLIs +> (`text_to_video.py` / `image_to_video.py`). Do **not** pass the Nano/Super numbers +> (720p / `guidance_scale` 6.0 / `flow_shift` 10.0) to Edge — they produce degenerate output. + For the larger checkpoints see [Cosmos3-Nano](./Cosmos3-Nano.md) and [Cosmos3-Super](./Cosmos3-Super.md). ## Support notes From 089bb88706d4e3038f370981a4d3d4b7e2910eec Mon Sep 17 00:00:00 2001 From: Alex Brooks Date: Thu, 30 Jul 2026 03:55:42 -0600 Subject: [PATCH 34/37] [Model] Remove Text Encoder Output Layer Hardcoding in Flux2 (#5589) Signed-off-by: Alex Brooks --- .../text_encoder/config.json | 30 ++----------------- .../flux2_klein/pipeline_flux2_klein.py | 19 +++++++----- 2 files changed, 14 insertions(+), 35 deletions(-) diff --git a/tests/model_tests/diffusion/tiny_configs/Flux2KleinPipeline/text_encoder/config.json b/tests/model_tests/diffusion/tiny_configs/Flux2KleinPipeline/text_encoder/config.json index 004914a7f..19a6b4063 100644 --- a/tests/model_tests/diffusion/tiny_configs/Flux2KleinPipeline/text_encoder/config.json +++ b/tests/model_tests/diffusion/tiny_configs/Flux2KleinPipeline/text_encoder/config.json @@ -13,31 +13,6 @@ "initializer_range": 0.02, "intermediate_size": 64, "layer_types": [ - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", "full_attention", "full_attention", "full_attention" @@ -46,7 +21,7 @@ "max_window_layers": 36, "model_type": "qwen3", "num_attention_heads": 2, - "num_hidden_layers": 28, + "num_hidden_layers": 3, "num_key_value_heads": 2, "rms_norm_eps": 1e-06, "rope_scaling": null, @@ -56,5 +31,6 @@ "transformers_version": "4.56.1", "use_cache": true, "use_sliding_window": false, - "vocab_size": 151936 + "vocab_size": 151936, + "text_encoder_out_layers":[0, 1, 2] } diff --git a/vllm_omni/diffusion/models/flux2_klein/pipeline_flux2_klein.py b/vllm_omni/diffusion/models/flux2_klein/pipeline_flux2_klein.py index 184b5d402..a4350373b 100644 --- a/vllm_omni/diffusion/models/flux2_klein/pipeline_flux2_klein.py +++ b/vllm_omni/diffusion/models/flux2_klein/pipeline_flux2_klein.py @@ -271,6 +271,15 @@ class Flux2KleinPipeline( self.tokenizer_max_length = 512 self.default_sample_size = 128 + # text_encoder_out_layers is hardcoded to (9, 18, 27) in Diffusers, + # but this won't necessarily work for all text encoders. We allow + # setting the text_encoder_out_layers in the config to allow + # custom text encoders + if hasattr(self.text_encoder.config, "text_encoder_out_layers"): + self.text_encoder_out_layers = self.text_encoder.config.text_encoder_out_layers + else: + self.text_encoder_out_layers = (9, 18, 27) + self._guidance_scale = None self._attention_kwargs = None self._num_timesteps = None @@ -293,10 +302,10 @@ class Flux2KleinPipeline( text_encoder: Qwen3ForCausalLM, tokenizer: Qwen2TokenizerFast, prompt: str | list[str], + hidden_states_layers: tuple[int, ...], dtype: torch.dtype | None = None, device: torch.device | None = None, max_sequence_length: int = 512, - hidden_states_layers: list[int] = (9, 18, 27), ): dtype = text_encoder.dtype if dtype is None else dtype device = text_encoder.device if device is None else device @@ -513,7 +522,6 @@ class Flux2KleinPipeline( num_images_per_prompt: int = 1, prompt_embeds: torch.Tensor | None = None, max_sequence_length: int = 512, - text_encoder_out_layers: tuple[int, ...] = (9, 18, 27), ): device = device or self._execution_device @@ -527,9 +535,9 @@ class Flux2KleinPipeline( text_encoder=self.text_encoder, tokenizer=self.tokenizer, prompt=prompt, + hidden_states_layers=self.text_encoder_out_layers, device=device, max_sequence_length=max_sequence_length, - hidden_states_layers=text_encoder_out_layers, ) batch_size, seq_len, _ = prompt_embeds.shape @@ -795,9 +803,6 @@ class Flux2KleinPipeline( latents = req.sampling_params.latents strength = req.sampling_params.strength if req.sampling_params.strength is not None else 1.0 - text_encoder_out_layers: tuple[int, ...] = req.sampling_params.extra_args.get( - "text_encoder_out_layers", (9, 18, 27) - ) padding_mask_crop: int | None = req.sampling_params.extra_args.get("padding_mask_crop") attention_kwargs: dict[str, Any] | None = None callback_on_step_end: Callable[[int, int, dict], None] | None = None @@ -833,7 +838,6 @@ class Flux2KleinPipeline( device=device, num_images_per_prompt=num_images_per_prompt, max_sequence_length=max_sequence_length, - text_encoder_out_layers=text_encoder_out_layers, ) if self.do_classifier_free_guidance: @@ -846,7 +850,6 @@ class Flux2KleinPipeline( device=device, num_images_per_prompt=num_images_per_prompt, max_sequence_length=max_sequence_length, - text_encoder_out_layers=text_encoder_out_layers, ) # 4. process images From 1752e8ee5b75e1169116560a6c2820b6fc9cc863 Mon Sep 17 00:00:00 2001 From: Yukim1 <121286183+zwhzzz0821@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:57:33 +0800 Subject: [PATCH 35/37] [Refactor] Migrate Qwen3 Omni thinker-only deploy config (#5285) Signed-off-by: zwhzzz0821 <2831474076@qq.com> --- .../examples/online_serving/qwen3_omni.md | 2 +- .../qwen3_omni/qwen3_omni_moe_thinking.yaml | 36 ----------------- .../qwen3_omni/run_gradio_demo.sh | 12 +++--- tests/config/test_config_factory.py | 39 +++++++++++++++++++ vllm_omni/deploy/qwen3_omni_moe_thinking.yaml | 21 ++++++++++ .../models/qwen3_omni/pipeline.py | 2 +- 6 files changed, 68 insertions(+), 44 deletions(-) delete mode 100644 examples/online_serving/qwen3_omni/qwen3_omni_moe_thinking.yaml create mode 100644 vllm_omni/deploy/qwen3_omni_moe_thinking.yaml diff --git a/docs/user_guide/examples/online_serving/qwen3_omni.md b/docs/user_guide/examples/online_serving/qwen3_omni.md index e372692a8..b4c05bdbe 100644 --- a/docs/user_guide/examples/online_serving/qwen3_omni.md +++ b/docs/user_guide/examples/online_serving/qwen3_omni.md @@ -306,7 +306,7 @@ The gradio script supports the following arguments: `````` ??? abstract "qwen3_omni_moe_thinking.yaml" ``````yaml - --8<-- "examples/online_serving/qwen3_omni/qwen3_omni_moe_thinking.yaml" + --8<-- "vllm_omni/deploy/qwen3_omni_moe_thinking.yaml" `````` ??? abstract "run_curl_multimodal_generation.sh" ``````sh diff --git a/examples/online_serving/qwen3_omni/qwen3_omni_moe_thinking.yaml b/examples/online_serving/qwen3_omni/qwen3_omni_moe_thinking.yaml deleted file mode 100644 index 7ce88f80a..000000000 --- a/examples/online_serving/qwen3_omni/qwen3_omni_moe_thinking.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# Stage config for running Qwen3-Omni-MoE-Thinking (text-only output) -# This config is for models like Qwen3-Omni-30B-A3B-Thinking that only have the -# thinker component and do not support audio output. -# -# Single stage: Thinker (multimodal understanding + text generation) - -# The following config has been verified on 2x H100-80G GPUs. -stage_args: - - stage_id: 0 - runtime: - devices: "0,1" - engine_args: - model_stage: thinker - max_num_seqs: 1 - model_arch: Qwen3OmniMoeForConditionalGeneration - worker_type: ar - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.9 - enforce_eager: true - trust_remote_code: true - engine_output_type: text - distributed_executor_backend: "mp" - enable_prefix_caching: true - hf_config_name: thinker_config - tensor_parallel_size: 2 - final_output: true - final_output_type: text - is_comprehension: true - default_sampling_params: - temperature: 0.4 - top_p: 0.9 - top_k: 1 - max_tokens: 2048 - seed: 42 - detokenize: True - repetition_penalty: 1.05 diff --git a/examples/online_serving/qwen3_omni/run_gradio_demo.sh b/examples/online_serving/qwen3_omni/run_gradio_demo.sh index 73ce273d9..465304534 100755 --- a/examples/online_serving/qwen3_omni/run_gradio_demo.sh +++ b/examples/online_serving/qwen3_omni/run_gradio_demo.sh @@ -13,7 +13,7 @@ set -e MODEL="Qwen/Qwen3-Omni-30B-A3B-Instruct" SERVER_PORT=8091 GRADIO_PORT=7861 -STAGE_CONFIGS_PATH="" +DEPLOY_CONFIG="" SERVER_HOST="0.0.0.0" GRADIO_IP="127.0.0.1" GRADIO_SHARE=false @@ -33,8 +33,8 @@ while [[ $# -gt 0 ]]; do GRADIO_PORT="$2" shift 2 ;; - --stage-configs-path) - STAGE_CONFIGS_PATH="$2" + --deploy-config) + DEPLOY_CONFIG="$2" shift 2 ;; --server-host) @@ -56,7 +56,7 @@ while [[ $# -gt 0 ]]; do echo " --model MODEL Model name/path (default: Qwen/Qwen3-Omni-30B-A3B-Instruct)" echo " --server-port PORT Port for vLLM server (default: 8091)" echo " --gradio-port PORT Port for Gradio demo (default: 7861)" - echo " --stage-configs-path PATH Path to custom stage configs YAML file (optional)" + echo " --deploy-config PATH Path to custom deploy config YAML file (optional)" echo " --server-host HOST Host for vLLM server (default: 0.0.0.0)" echo " --gradio-ip IP IP for Gradio demo (default: 127.0.0.1)" echo " --share Share Gradio demo publicly" @@ -87,8 +87,8 @@ echo "==========================================" # Build vLLM server command SERVER_CMD=("vllm" "serve" "$MODEL" "--omni" "--port" "$SERVER_PORT" "--host" "$SERVER_HOST") -if [ -n "$STAGE_CONFIGS_PATH" ]; then - SERVER_CMD+=("--stage-configs-path" "$STAGE_CONFIGS_PATH") +if [ -n "$DEPLOY_CONFIG" ]; then + SERVER_CMD+=("--deploy-config" "$DEPLOY_CONFIG") fi # Function to cleanup on exit diff --git a/tests/config/test_config_factory.py b/tests/config/test_config_factory.py index 3e482f48f..a811868c1 100644 --- a/tests/config/test_config_factory.py +++ b/tests/config/test_config_factory.py @@ -1233,6 +1233,45 @@ stages: assert deploy.connectors is not None assert deploy.platforms is not None + def test_qwen3_omni_thinker_only_deploy_config_uses_resolved_pipeline(self): + deploy_path = Path(get_deploy_config_path("qwen3_omni_moe_thinking.yaml")) + with patch( + "vllm_omni.config.config_factory.get_config", + return_value=Q3_OMNI_THINKER_HF_CONFIG, + ): + pipeline = StageConfigFactory.get_pipeline_config( + "Qwen/Qwen3-Omni-30B-A3B-Thinking", + trust_remote_code=False, + deploy_config_path=str(deploy_path), + ) + + assert pipeline is not None + assert pipeline.model_type == "qwen3_omni_moe_thinker_only" + + deploy = load_deploy_config(deploy_path) + stages = merge_pipeline_deploy(pipeline, deploy) + + assert len(stages) == 1 + thinker = stages[0] + assert thinker.model_stage == "thinker" + assert thinker.final_output is True + assert thinker.final_output_type == "text" + assert thinker.worker_type == "ar" + assert thinker.scheduler_cls == "vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler" + assert thinker.hf_config_name == "thinker_config" + assert thinker.is_comprehension is True + assert thinker.yaml_runtime["devices"] == "0,1" + assert thinker.yaml_engine_args["engine_output_type"] == "text" + assert thinker.yaml_engine_args["tensor_parallel_size"] == 2 + assert thinker.yaml_engine_args["max_num_seqs"] == 1 + assert thinker.yaml_engine_args["gpu_memory_utilization"] == 0.9 + assert thinker.yaml_engine_args["enforce_eager"] is True + assert thinker.yaml_engine_args["async_scheduling"] is False + assert thinker.yaml_engine_args["distributed_executor_backend"] == "mp" + assert thinker.yaml_engine_args["enable_prefix_caching"] is True + assert thinker.yaml_engine_args["async_chunk"] is False + assert thinker.yaml_extras["default_sampling_params"]["detokenize"] is True + def test_load_voxtral_tts_deploy_config_schema_fields(self): deploy_path = Path(get_deploy_config_path("voxtral_tts.yaml")) deploy = load_deploy_config(deploy_path) diff --git a/vllm_omni/deploy/qwen3_omni_moe_thinking.yaml b/vllm_omni/deploy/qwen3_omni_moe_thinking.yaml new file mode 100644 index 000000000..7ccc64750 --- /dev/null +++ b/vllm_omni/deploy/qwen3_omni_moe_thinking.yaml @@ -0,0 +1,21 @@ +# Deploy config for Qwen3-Omni-MoE thinker-only checkpoints. +# Verified on 2x H100-80G GPUs. +pipeline: qwen3_omni_moe +distributed_executor_backend: mp +enable_prefix_caching: true + +stages: + - stage_id: 0 + devices: "0,1" + max_num_seqs: 1 + gpu_memory_utilization: 0.9 + enforce_eager: true + async_scheduling: false + tensor_parallel_size: 2 + default_sampling_params: + temperature: 0.4 + top_p: 0.9 + top_k: 1 + max_tokens: 2048 + seed: 42 + repetition_penalty: 1.05 diff --git a/vllm_omni/model_executor/models/qwen3_omni/pipeline.py b/vllm_omni/model_executor/models/qwen3_omni/pipeline.py index 9baecdd3e..c93e764ef 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/pipeline.py +++ b/vllm_omni/model_executor/models/qwen3_omni/pipeline.py @@ -88,7 +88,7 @@ QWEN3_OMNI_THINKER_ONLY_PIPELINE = PipelineConfig( owns_tokenizer=True, requires_multimodal_data=True, hf_config_name="thinker_config", - engine_output_type="latent", + engine_output_type="text", sampling_constraints={"detokenize": True}, ), ), From b3f4fbf9e217ab6174fc0c475e33a478006ac34c Mon Sep 17 00:00:00 2001 From: Yukim1 <121286183+zwhzzz0821@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:59:05 +0800 Subject: [PATCH 36/37] [Refactor] Migrate Step-Audio2 offline test to deploy config (#5319) Signed-off-by: zwhzzz0821 <2831474076@qq.com> --- .../test_step_audio2_expansion.py | 4 +- vllm_omni/deploy/step_audio2_thinker_ci.yaml | 37 ------------------- 2 files changed, 2 insertions(+), 39 deletions(-) delete mode 100644 vllm_omni/deploy/step_audio2_thinker_ci.yaml diff --git a/tests/e2e/offline_inference/test_step_audio2_expansion.py b/tests/e2e/offline_inference/test_step_audio2_expansion.py index 00fc7bbe2..cac4fbe52 100644 --- a/tests/e2e/offline_inference/test_step_audio2_expansion.py +++ b/tests/e2e/offline_inference/test_step_audio2_expansion.py @@ -18,8 +18,8 @@ from tests.helpers.stage_config import get_deploy_config_path from vllm_omni.outputs import OmniRequestOutput MODEL = "stepfun-ai/Step-Audio-2-mini" -STAGE_CONFIG = get_deploy_config_path("step_audio2_thinker_ci.yaml") -TEST_PARAMS = [(MODEL, STAGE_CONFIG)] +DEPLOY_CONFIG = get_deploy_config_path("step_audio2_ci.yaml") +TEST_PARAMS = [(MODEL, None, {"deploy_config": DEPLOY_CONFIG, "trust_remote_code": True})] pytestmark = [pytest.mark.slow, pytest.mark.tts] diff --git a/vllm_omni/deploy/step_audio2_thinker_ci.yaml b/vllm_omni/deploy/step_audio2_thinker_ci.yaml deleted file mode 100644 index 7856ebda9..000000000 --- a/vllm_omni/deploy/step_audio2_thinker_ci.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# Stage config for Step-Audio2 CI tests (offline inference, thinker-only). -# Single Thinker stage (audio/text understanding → text). ASR/text-output -# tests don't need Token2Wav; a two-stage config would deadlock on stage-1 skip. - -# The following config has been verified on 2x 24GB GPU (L4/RTX3090/RTX4090). -stage_args: - - stage_id: 0 - runtime: - process: true - devices: "0" - max_batch_size: 1 - engine_args: - model_stage: thinker - model_arch: StepAudio2ThinkerForConditionalGeneration - tensor_parallel_size: 1 - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - max_model_len: 4096 - max_num_batched_tokens: 4096 - max_num_seqs: 1 - gpu_memory_utilization: 0.8 - skip_mm_profiling: true - enforce_eager: true - trust_remote_code: true - enable_prefix_caching: false - engine_output_type: text # Output text + audio tokens - is_comprehension: true - final_output: true - final_output_type: text - default_sampling_params: - temperature: 0.7 - top_p: 0.9 - top_k: -1 - max_tokens: 256 - seed: 42 - detokenize: true - repetition_penalty: 1.05 From 744c65b730e8820732610161efe0aa2375825799 Mon Sep 17 00:00:00 2001 From: evanchueng Date: Thu, 30 Jul 2026 21:35:38 +0800 Subject: [PATCH 37/37] =?UTF-8?q?[Feature]=20Distributed=20layerwise=20off?= =?UTF-8?q?load=20with=20DP=20multi-concurrency=20+=20mmap=20weight?= =?UTF-8?q?=E2=80=A6=20(#5397)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: evanchueng Signed-off-by: david6666666 <530634352@qq.com> Co-authored-by: david6666666 <530634352@qq.com> --- .../diffusion/cpu_offload_diffusion.md | 162 +- recipes/cosmos3/Cosmos3-DistOffload.md | 191 +++ .../test_distributed_layerwise_backend.py | 1027 ++++++++++++ tests/diffusion/test_diffusion_engine.py | 17 +- .../test_multiproc_engine_concurrency.py | 24 +- vllm_omni/config/omni_config.py | 2 + vllm_omni/diffusion/data.py | 6 + vllm_omni/diffusion/diffusion_engine.py | 61 +- .../diffusion/executor/multiproc_executor.py | 164 +- vllm_omni/diffusion/ipc.py | 19 +- .../model_loader/diffusers_loader.py | 45 +- vllm_omni/diffusion/offloader/__init__.py | 32 + vllm_omni/diffusion/offloader/base.py | 70 +- .../diffusion/offloader/block_discovery.py | 85 + .../distributed_layerwise_backend.py | 1491 +++++++++++++++++ .../diffusion/offloader/module_collector.py | 2 + vllm_omni/diffusion/offloader/offload_plan.py | 61 + vllm_omni/diffusion/offloader/tensor_utils.py | 47 + .../worker/diffusion_model_runner.py | 9 +- .../diffusion/worker/diffusion_worker.py | 121 +- vllm_omni/engine/arg_utils.py | 2 + vllm_omni/engine/async_omni_engine.py | 2 + vllm_omni/entrypoints/cli/serve.py | 24 + 23 files changed, 3586 insertions(+), 78 deletions(-) create mode 100644 recipes/cosmos3/Cosmos3-DistOffload.md create mode 100644 tests/diffusion/offloader/test_distributed_layerwise_backend.py create mode 100644 vllm_omni/diffusion/offloader/block_discovery.py create mode 100644 vllm_omni/diffusion/offloader/distributed_layerwise_backend.py create mode 100644 vllm_omni/diffusion/offloader/offload_plan.py create mode 100644 vllm_omni/diffusion/offloader/tensor_utils.py diff --git a/docs/user_guide/diffusion/cpu_offload_diffusion.md b/docs/user_guide/diffusion/cpu_offload_diffusion.md index 31a2670ba..86a0e9201 100644 --- a/docs/user_guide/diffusion/cpu_offload_diffusion.md +++ b/docs/user_guide/diffusion/cpu_offload_diffusion.md @@ -179,7 +179,134 @@ class Flux2Transformer2DModel(nn.Module): - Support single GPU only for now -### Implementation Notes +## Distributed Layerwise Offloading + +### How It Works + +Distributed layerwise offloading extends single-GPU layerwise offloading to +multi-device deployments. Each DP rank stores only **1/dp_size** of the model +weights in host memory; full layer weights are reconstructed at runtime via +**AllGather** on a dedicated communication stream, overlapped with computation +via a fixed double-buffer scheme. + +**Key features:** +- **Weight sharding + AllGather**: each rank stores 1/dp_size of weights, + reconstructed per-layer via `all_gather_into_tensor` +- **Fixed double-buffer**: exactly 2 transformer blocks on each device at any + time, regardless of model size +- **DP multi-concurrency**: N concurrent requests processed in parallel + (AllGather only gathers weight shards, which are request-independent) +- **mmap weight loading**: weights loaded as mmap views pointing to shared OS + page cache, eliminating O(dp_size × model_size) RSS during model creation +- **Platform-agnostic**: works on NVIDIA GPU (CUDA/NCCL) and Ascend NPU + (CANN/HCCL) via vLLM-Omni's platform abstraction + +**Execution Flow (per device):** + +``` +Compute Stream: [Layer N] [Layer N+1] [Layer N+2] +H2D Stream: [H2D Shard N+1] [H2D Shard N+2] +AllGather: [AG N+1] [AG N+2] +Slot Usage: Slot0: Layer N Slot1: Layer N+1 +``` + +### Usage + +**CLI:** +```bash +# 4× GPU/NPU with AllGather (recommended) +vllm serve /path/to/model --omni \ + --enable-distributed-layerwise-offload \ + --data-parallel-size 4 + +# Without AllGather (each rank loads full weights, no sharding) +vllm serve /path/to/model --omni \ + --enable-distributed-layerwise-offload \ + --data-parallel-size 4 \ + --dlo-no-use-allgather + +# With SP instead of DP (long sequences) +vllm serve /path/to/model --omni \ + --enable-distributed-layerwise-offload \ + --usp 4 +``` + +**Python API:** +```python +from vllm_omni import Omni + +m = Omni( + model="/path/to/model", + enable_distributed_layerwise_offload=True, + dlo_use_allgather=True, # default +) +``` + +### CLI Flags + +| Flag | Description | Default | +|------|-------------|---------| +| `--enable-distributed-layerwise-offload` | Enable DLO | `false` | +| `--data-parallel-size N` | Number of DP ranks for weight sharding | `1` | +| `--dlo-use-allgather` | Shard + AllGather (saves CPU, requires concurrent requests) | `true` | +| `--dlo-no-use-allgather` | Full weights per rank (no sharding, no AllGather) | `false` | + +### How model weights are loaded (mmap path) + +When DLO + AllGather is active, the offloader: + +1. **Saves non-persistent buffers** (e.g. RoPE `inv_freq`, timestep `freqs`) + from the normally-created transformer +2. **Converts to meta device** via `to_empty(device="meta")`, releasing random + initialization weights +3. **Loads checkpoint weights as mmap views** via `safe_open().get_tensor()`, + which return views into the OS page cache (shared across all ranks, 0 RSS) +4. **Calls `post_load_weights()`** to apply model-specific dtype conversions + (e.g. Cosmos3's `time_embedder` → FP32) +5. **Restores non-persistent buffers** from saved copies + +This approach requires **zero model-specific code changes** — no pipeline or +transformer modifications are needed. + +### OffloadPlan (declarative topology metadata) + +Models can optionally declare an `OffloadPlan` class variable to provide +topology metadata (block attribute names, submodules to offload) without +any offload-specific logic: + +```python +from vllm_omni.diffusion.offloader import OffloadPlan + +class MyPipeline(nn.Module): + _dit_modules = ["transformer"] + _offload_plan = OffloadPlan( + block_attrs={"transformer": ("blocks",)}, + offload_submodules={"context_encoder": "layers"}, + ) +``` + +When not declared, the offloader falls back to `_layerwise_offload_blocks_attrs` +and heuristic attribute search. This is backward-compatible — existing models +work without any changes. + +### DP Multi-concurrency + +When `--data-parallel-size > 1` and AllGather is enabled, the scheduler batches +up to `dp_size` requests per denoise step. Each DP rank processes a different +request while AllGather synchronizes only weight shards (request-independent). + +**Requirements for concurrent requests:** +- `num_inference_steps` must be specified explicitly (None is not allowed) +- All concurrent requests must have the same `num_inference_steps` value +- These constraints exist because AllGather is a collective that requires + every rank to participate at each denoise step + +### Limitations +- Online quantization (FP8) is incompatible with mmap loading — falls back to + regular `load_weights()` automatically +- Tensor Parallel is not supported (DLO uses DP-based sharding) +- HSDP + AllGather is rejected (would double-shard weights) +- `num_inference_steps=None` is not allowed in DP multi-concurrency mode **Module Discovery** @@ -208,7 +335,10 @@ OffloadBackend (base class) ├── ModelLevelOffloadBackend → uses SequentialOffloadHook (.to() swap) │ (delegates to a pipeline's enable_omni_model_cpu_offload │ for split models like Cosmos3) -└── LayerWiseOffloadBackend → uses LayerwiseOffloadHook +├── LayerWiseOffloadBackend → uses LayerwiseOffloadHook +│ (single-GPU, full weights on host) +└── DistributedLayerwiseOffloadBackend → uses DistributedLayerwiseOffloadHook + (multi-GPU, 1/dp_size sharded weights + AllGather) ``` Factory function `get_offload_backend()` selects the appropriate backend based on @@ -221,20 +351,22 @@ reasoner/generator components inside the model forward pass. ## Supported Models -| Architecture | Example Models | DiT Class | Model-Level Offload | Layerwise Offload | Blocks Attrs (Layerwise specific) | -|--------------|----------------|-----------|---------------------|-------------------|-----------------------------------| -| Flux2Pipeline | `black-forest-labs/FLUX.2-dev` | `Flux2Transformer2DModel` | ✓ | ✓ | `"transformer_blocks"`, `"single_transformer_blocks"` | -| LongCatImagePipeline | `meituan-longcat/LongCat-Image` | `LongCatImageTransformer2DModel` | - | ✓ | `"transformer_blocks"`, `"single_transformer_blocks"` | -| NextStep11Pipeline | `stepfun-ai/NextStep-1.1` | `NextStepModel` | - | ✓ | `"layers"` | -| OvisImagePipeline | `AIDC-AI/Ovis-Image-7B` | `OvisImageTransformer2DModel` | - | ✓ | `"transformer"` | -| QwenImagePipeline | `Qwen/Qwen-Image` | `QwenImageTransformer2DModel` | ✓ | ✓ | `"transformer_blocks"` | -| StableDiffusionXLPipeline | `stabilityai/stable-diffusion-xl-base-1.0` | `SDXLUNet2DConditionModel` | ✓ | ✓ | `"down_blocks"`, `"up_blocks"` | -| StableDiffusion3Pipeline | `stabilityai/stable-diffusion-3.5-medium` | `SD3Transformer2DModel` | - | ✓ | `"transformer_blocks"` | -| Wan22I2VPipeline | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | `"blocks"` | -| Wan22Pipeline | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | `"blocks"` | -| SoulXSingerPipeline / SoulXSingerSVCPipeline | `Soul-AILab/SoulX-Singer` | `DiffLlama` (`cfm_decoder.model.diff_estimator`) | ✓ | ✓ | `"layers"` | -| BagelPipeline | `ByteDance-Seed/BAGEL-7B-MoT` | `Qwen2MoTModel` | - | ✓ | `"layers"`, `"customized modules"` | +| Architecture | Example Models | DiT Class | Model-Level Offload | Layerwise Offload | Distributed Layerwise Offload | Blocks Attrs (Layerwise specific) | +|--------------|----------------|-----------|---------------------|-------------------|-------------------------------|-----------------------------------| +| Flux2Pipeline | `black-forest-labs/FLUX.2-dev` | `Flux2Transformer2DModel` | ✓ | ✓ | - | `"transformer_blocks"`, `"single_transformer_blocks"` | +| LongCatImagePipeline | `meituan-longcat/LongCat-Image` | `LongCatImageTransformer2DModel` | - | ✓ | - | `"transformer_blocks"`, `"single_transformer_blocks"` | +| NextStep11Pipeline | `stepfun-ai/NextStep-1.1` | `NextStepModel` | - | ✓ | - | `"layers"` | +| OvisImagePipeline | `AIDC-AI/Ovis-Image-7B` | `OvisImageTransformer2DModel` | - | ✓ | - | `"transformer"` | +| QwenImagePipeline | `Qwen/Qwen-Image` | `QwenImageTransformer2DModel` | ✓ | ✓ | - | `"transformer_blocks"` | +| StableDiffusionXLPipeline | `stabilityai/stable-diffusion-xl-base-1.0` | `SDXLUNet2DConditionModel` | ✓ | ✓ | - | `"down_blocks"`, `"up_blocks"` | +| StableDiffusion3Pipeline | `stabilityai/stable-diffusion-3.5-medium` | `SD3Transformer2DModel` | - | ✓ | - | `"transformer_blocks"` | +| Wan22I2VPipeline | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | - | `"blocks"` | +| Wan22Pipeline | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | - | `"blocks"` | +| SoulXSingerPipeline / SoulXSingerSVCPipeline | `Soul-AILab/SoulX-Singer` | `DiffLlama` (`cfm_decoder.model.diff_estimator`) | ✓ | ✓ | - | `"layers"` | +| BagelPipeline | `ByteDance-Seed/BAGEL-7B-MoT` | `Qwen2MoTModel` | - | ✓ | - | `"layers"`, `"customized modules"` | +| Cosmos3OmniDiffusersPipeline | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | `Cosmos3VFMTransformer`, `Cosmos3LanguageModel` | ✓ | ✓ | ✓ | `"layers"`, `"gen_layers"` | **Notes:** - Model-Level Offloading is expected to be supported by all common diffusion models (DiT and encoders) naturally - Layerwise Offloading requires DiT class to define `_layerwise_offload_blocks_attrs` pointing to transformer blocks +- Distributed Layerwise Offloading works with any model that supports Layerwise Offloading — no additional model changes required. See [Cosmos3 DistOffload recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/cosmos3/Cosmos3-DistOffload.md) for usage examples. diff --git a/recipes/cosmos3/Cosmos3-DistOffload.md b/recipes/cosmos3/Cosmos3-DistOffload.md new file mode 100644 index 000000000..27d1c77bd --- /dev/null +++ b/recipes/cosmos3/Cosmos3-DistOffload.md @@ -0,0 +1,191 @@ +# Cosmos3 — Distributed Layerwise Offload + +> Sharding weights across DP ranks with H2D + AllGather overlap for multi-NPU/GPU deployment + +## When to use this recipe + +Use this recipe when deploying Cosmos3-Super (64B / 124 GB) or Cosmos3-Nano (17B / 33 GB) across multiple devices where the model does not fit on a single device's HBM, or when you need higher throughput via DP multi-concurrency. + +**Key benefits:** +- Each rank stores only 1/dp_size of the model weights on host memory +- Only 2 transformer blocks reside on each device at any time (fixed double-buffer) +- DP multi-concurrency: N concurrent requests with near-linear throughput scaling +- mmap weight loading: shared page cache eliminates O(dp_size × model_size) RSS + +## Prerequisites + +- 2+ NPU or GPU devices (Ascend 910B3, NVIDIA B300, etc.) +- vLLM-Omni with distributed layerwise offload support +- Cosmos3-Nano or Cosmos3-Super checkpoint (local path or HuggingFace repo ID) + +## Serve commands + +### Cosmos3-Nano with DP=4 + AllGather (recommended for max throughput) + +```bash +vllm serve /path/to/Cosmos3-Nano \ + --omni \ + --host 0.0.0.0 --port 8000 \ + --no-guardrails --init-timeout 3600 \ + --vae-use-tiling --vae-patch-parallel-size 1 \ + --enable-distributed-layerwise-offload \ + --data-parallel-size 4 +``` + +### Cosmos3-Super with DP=2 + AllGather (for models > single-card HBM) + +```bash +vllm serve /path/to/Cosmos3-Super \ + --omni \ + --host 0.0.0.0 --port 8000 \ + --no-guardrails --init-timeout 3600 \ + --vae-use-tiling --vae-patch-parallel-size 1 \ + --enable-distributed-layerwise-offload \ + --data-parallel-size 2 +``` + +### Cosmos3-Super with DP=4 + AllGather + +```bash +vllm serve /path/to/Cosmos3-Super \ + --omni \ + --host 0.0.0.0 --port 8000 \ + --no-guardrails --init-timeout 3600 \ + --vae-use-tiling --vae-patch-parallel-size 1 \ + --enable-distributed-layerwise-offload \ + --data-parallel-size 4 +``` + +### Without AllGather (full weights per rank, no sharding) + +Use `--dlo-no-use-allgather` when you want each rank to load full weights independently — +no AllGather synchronization, no concurrent request requirement, but N× host memory. + +```bash +vllm serve /path/to/Cosmos3-Nano \ + --omni \ + --host 0.0.0.0 --port 8000 \ + --no-guardrails --init-timeout 3600 \ + --vae-use-tiling --vae-patch-parallel-size 1 \ + --enable-distributed-layerwise-offload \ + --data-parallel-size 4 \ + --dlo-no-use-allgather +``` + +### With SP (sequence parallel) instead of DP + +For long-sequence workloads where SP parallelism is preferred: + +```bash +vllm serve /path/to/Cosmos3-Super \ + --omni \ + --host 0.0.0.0 --port 8000 \ + --no-guardrails --init-timeout 3600 \ + --vae-use-tiling --vae-patch-parallel-size 1 \ + --enable-distributed-layerwise-offload \ + --usp 4 +``` + +## Request examples + +### Text-to-Image (T2I) + +```bash +curl -s -o output.png -X POST "http://localhost:8000/v1/images/generations" \ + -H "Content-Type: multipart/form-data" \ + -F "model=/path/to/Cosmos3-Nano" \ + -F "prompt=A robot arm cleaning a plate, cinematic" \ + -F "size=1024x1024" \ + -F "num_inference_steps=50" \ + -F "guidance_scale=6.0" \ + -F "seed=42" +``` + +### Text-to-Video (T2V) + +```bash +curl -s -o output.mp4 -X POST "http://localhost:8000/v1/videos/sync" \ + -H "Accept: video/mp4" \ + -F "model=/path/to/Cosmos3-Nano" \ + -F "prompt=A robot arm cleaning a plate, cinematic shot" \ + -F "negative_prompt=blurry" \ + -F "size=832x480" \ + -F "num_frames=29" \ + -F "fps=24" \ + -F "num_inference_steps=35" \ + -F "guidance_scale=6.0" \ + -F "max_sequence_length=4096" \ + -F "flow_shift=10.0" \ + -F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":false}' \ + -F "seed=17" +``` + +> **Important:** When using DP multi-concurrency (AllGather mode), `num_inference_steps` +> must be specified explicitly and must be the same for all concurrent requests. +> `num_inference_steps=None` is not allowed because it may resolve differently +> per request mode (e.g., T2V=35 vs action_mode=30), which would deadlock AllGather. + +### Multiple concurrent requests (DP multi-concurrency) + +With `--data-parallel-size 4`, send 4 concurrent requests to maximize throughput: + +```bash +for i in 1 2 3 4; do + curl -s -o output_${i}.mp4 -X POST "http://localhost:8000/v1/videos/sync" \ + -H "Accept: video/mp4" \ + -F "model=/path/to/Cosmos3-Nano" \ + -F "prompt=A robot arm cleaning a plate, shot ${i}" \ + -F "negative_prompt=blurry" \ + -F "size=832x480" \ + -F "num_frames=29" \ + -F "fps=24" \ + -F "num_inference_steps=35" \ + -F "guidance_scale=6.0" \ + -F "max_sequence_length=4096" \ + -F "flow_shift=10.0" \ + -F 'extra_params={"use_resolution_template":false,"use_duration_template":false,"guardrails":false}' \ + -F "seed=17" & +done +wait +``` + +## CLI flags + +| Flag | Description | Default | +|------|-------------|---------| +| `--enable-distributed-layerwise-offload` | Enable DLO with H2D + AllGather overlap | `false` | +| `--data-parallel-size N` | Number of DP ranks (weight sharding + concurrent requests) | `1` | +| `--dlo-use-allgather` | Use shard + AllGather for weight reconstruction (recommended) | `true` | +| `--dlo-no-use-allgather` | Each rank loads full weights independently (no sharding, no AllGather) | `false` | +| `--usp N` | Use SP instead of DP for weight sharding (long sequences) | `1` | + +## Mode comparison + +| Mode | CLI flags | CPU/rank | HBM/card | Throughput | Use case | +|------|----------|---------|---------|-----------|----------| +| DLO + AllGather (DP4) | `--enable-distributed-layerwise-offload --data-parallel-size 4` | 1/4 model | 2 blocks | **3.3× HSDP** | Max throughput, short sequences | +| DLO + AllGather (DP2) | `--enable-distributed-layerwise-offload --data-parallel-size 2` | 1/2 model | 2 blocks | 2× HSDP | Balanced throughput + memory | +| DLO + AllGather (SP4) | `--enable-distributed-layerwise-offload --usp 4` | 1/4 model | 2 blocks | 1× (single req) | Long sequences (720p+) | +| DLO no-AllGather | `--enable-distributed-layerwise-offload --dlo-no-use-allgather` | Full model | 2 blocks | 0.4× HSDP | No AllGather overhead, high CPU | +| HSDP | `--use-hsdp --hsdp-shard-size 4` | 0 | 1/4 model | 1× (baseline) | Weights fit in HBM | + +## Memory expectations + +| Model | Config | Host RAM (cgroup) | HBM/card | +|-------|--------|-------------------|----------| +| Nano (33 GB) | DP2 + AllGather | ~38 GB | ~10 GB | +| Nano (33 GB) | DP4 + AllGather | ~47 GB | ~10 GB | +| Super (124 GB) | DP2 + AllGather | ~157 GB | ~15 GB | +| Super (124 GB) | DP4 + AllGather | ~172 GB | ~10 GB | + +> Host RAM is dominated by page cache (1× model_size, shared across ranks). +> HBM only holds 2 transformer blocks + framework + activations, independent of model size. + +## Limitations + +- `--dlo-use-allgather` (default) requires all concurrent requests to have the same + `num_inference_steps` (AllGather is a collective that needs all ranks synchronized) +- Online quantization (FP8) is incompatible with mmap loading — falls back to regular + `load_weights()` automatically +- Tensor Parallel is not supported (DLO uses DP-based sharding, a different dimension) +- HSDP + AllGather is rejected (would double-shard weights) diff --git a/tests/diffusion/offloader/test_distributed_layerwise_backend.py b/tests/diffusion/offloader/test_distributed_layerwise_backend.py new file mode 100644 index 000000000..c02fdaabc --- /dev/null +++ b/tests/diffusion/offloader/test_distributed_layerwise_backend.py @@ -0,0 +1,1027 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Unit tests for DistributedLayerwiseOffloadHook and backend utilities.""" + +import gc +import json +import os +import socket +from contextlib import contextmanager +from types import SimpleNamespace + +import pytest +import torch +import torch.distributed as dist +from safetensors.torch import save_file +from torch import nn +from torch.distributed.tensor import DeviceMesh, DTensor, Replicate + +import vllm_omni.diffusion.offloader.distributed_layerwise_backend as dist_backend_module +from vllm_omni.diffusion.offloader.base import OffloadConfig, OffloadStrategy +from vllm_omni.diffusion.offloader.block_discovery import ( + get_blocks_attr_names, + get_blocks_from_dit, + set_blocks_attr_names, +) +from vllm_omni.diffusion.offloader.distributed_layerwise_backend import ( + DistributedLayerwiseOffloadBackend, + DistributedLayerwiseOffloadHook, +) +from vllm_omni.diffusion.offloader.offload_plan import OffloadPlan, get_offload_plan +from vllm_omni.platforms import current_omni_platform + +pytestmark = [pytest.mark.diffusion, pytest.mark.cpu, pytest.mark.core_model] + + +class DummyStream: + def wait_stream(self, _stream) -> None: + return None + + def wait_event(self, _event) -> None: + return None + + +class DummyEvent: + def record(self, _stream) -> None: + return None + + +@contextmanager +def dummy_stream(_stream): + yield None + + +def _find_free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return int(s.getsockname()[1]) + + +def _set_dist_env(*, rank: int, world_size: int, master_port: int) -> None: + os.environ["RANK"] = str(rank) + os.environ["LOCAL_RANK"] = str(rank) + os.environ["WORLD_SIZE"] = str(world_size) + os.environ["MASTER_ADDR"] = "127.0.0.1" + os.environ["MASTER_PORT"] = str(master_port) + + +def _cleanup_distributed() -> None: + if dist.is_initialized(): + dist.destroy_process_group() + + for key in ["MASTER_ADDR", "MASTER_PORT", "RANK", "WORLD_SIZE", "LOCAL_RANK"]: + os.environ.pop(key, None) + + gc.collect() + if current_omni_platform.is_available(): + current_omni_platform.empty_cache() + current_omni_platform.synchronize() + + +@pytest.fixture(scope="module") +def dist_group(): + master_port = _find_free_port() + _set_dist_env(rank=0, world_size=1, master_port=master_port) + + dist.init_process_group("gloo", rank=0, world_size=1) + try: + yield + finally: + _cleanup_distributed() + + +@pytest.fixture +def patched_offload_runtime(monkeypatch): + monkeypatch.setattr(dist_backend_module.current_omni_platform, "Stream", DummyStream) + monkeypatch.setattr(dist_backend_module.current_omni_platform, "Event", DummyEvent) + monkeypatch.setattr(dist_backend_module.current_omni_platform, "current_stream", lambda: DummyStream()) + monkeypatch.setattr(dist_backend_module.current_omni_platform, "stream", dummy_stream) + + +class TinyBlock(nn.Module): + def __init__(self, values: torch.Tensor): + super().__init__() + mesh = DeviceMesh("cpu", [0]) + dtensor = DTensor.from_local(values, mesh, [Replicate()]) + self.weight = nn.Parameter(dtensor) + + +def _make_values(start: float) -> torch.Tensor: + return torch.arange(start, start + 4, dtype=torch.float32) + + +class TestDistributedLayerwiseOffloadHook: + def test_shard_and_pin_single_rank(self, dist_group, patched_offload_runtime): + """With dp_size=1, the shard should equal the full weights.""" + current_block = TinyBlock(_make_values(1.0)) + next_block = TinyBlock(_make_values(10.0)) + + hook = DistributedLayerwiseOffloadHook( + next_block=next_block, + device=torch.device("cpu"), + dp_group=None, + dp_size=1, + rank=0, + copy_stream=DummyStream(), + comm_stream=DummyStream(), + pin_memory=False, + ) + + hook.initialize_hook(current_block) + + # With dp_size=1, the CPU shard should contain all 4 elements + assert next_block.weight.dtype in hook.cpu_shards + shard = hook.cpu_shards[next_block.weight.dtype] + assert shard.numel() == 4 + assert torch.equal(shard, _make_values(10.0)) + + def test_dtensor_wrapper_preserved_across_prefetch_and_offload(self, dist_group, patched_offload_runtime): + """DTensor wrapper should be preserved through prefetch/offload cycle.""" + current_block = TinyBlock(_make_values(1.0)) + next_block = TinyBlock(_make_values(10.0)) + + hook = DistributedLayerwiseOffloadHook( + next_block=next_block, + device=torch.device("cpu"), + dp_group=None, + dp_size=1, + rank=0, + copy_stream=DummyStream(), + comm_stream=DummyStream(), + pin_memory=False, + ) + + hook.initialize_hook(current_block) + + # After init, next_block weights should be placeholders + assert isinstance(next_block.weight, DTensor) + assert next_block.weight.to_local().is_meta + + # Prefetch into slot 0 + hook.prefetch_layer(slot=0, non_blocking=False) + + # After prefetch, next_block weights should be materialized + assert isinstance(next_block.weight, DTensor) + assert torch.equal(next_block.weight.to_local(), _make_values(10.0)) + + # Offload current block + hook.offload_layer() + assert isinstance(current_block.weight, DTensor) + assert current_block.weight.to_local().is_meta + assert not hook.is_materialized + + def test_double_buffer_slot_swapping(self, dist_group, patched_offload_runtime): + """Verify slot swapping works correctly after each layer.""" + current_block = TinyBlock(_make_values(1.0)) + next_block = TinyBlock(_make_values(10.0)) + + hook = DistributedLayerwiseOffloadHook( + next_block=next_block, + device=torch.device("cpu"), + dp_group=None, + dp_size=1, + rank=0, + copy_stream=DummyStream(), + comm_stream=DummyStream(), + pin_memory=False, + ) + + hook.initialize_hook(current_block) + + assert hook.current_slot == 0 + + # Prefetch into slot 1 (next slot) + hook.prefetch_layer(slot=1, non_blocking=False) + + # Simulate post_forward: offload + swap + hook.offload_layer() + hook.current_slot = 1 - hook.current_slot + + assert hook.current_slot == 1 + + # Next iteration: prefetch into slot 0 + hook.prefetch_layer(slot=0, non_blocking=False) + hook.offload_layer() + hook.current_slot = 1 - hook.current_slot + + assert hook.current_slot == 0 + + def test_device_buffers_allocated(self, dist_group, patched_offload_runtime): + """Verify exactly two device buffers are allocated.""" + current_block = TinyBlock(_make_values(1.0)) + next_block = TinyBlock(_make_values(10.0)) + + hook = DistributedLayerwiseOffloadHook( + next_block=next_block, + device=torch.device("cpu"), + dp_group=None, + dp_size=1, + rank=0, + pin_memory=False, + ) + + hook.initialize_hook(current_block) + + # Two slots should be allocated + assert hook.gpu_buffers[0] is not None + assert hook.gpu_buffers[1] is not None + + # Each slot should have one dtype entry (float32) + assert len(hook.gpu_buffers[0]) == 1 + assert len(hook.gpu_buffers[1]) == 1 + + # Buffer size should match total numel + dtype = next_block.weight.dtype + assert hook.gpu_buffers[0][dtype].numel() == 4 + assert hook.gpu_buffers[1][dtype].numel() == 4 + + def test_sharding_multiple_ranks(self, dist_group, patched_offload_runtime): + """Verify weight sharding splits correctly across ranks.""" + block = nn.Module() + block.weight = nn.Parameter(torch.arange(8, dtype=torch.float32)) + + next_block = nn.Module() + next_block.weight = nn.Parameter(torch.arange(100, 108, dtype=torch.float32)) + + # Simulate dp_size=4, rank=1 + hook = DistributedLayerwiseOffloadHook( + next_block=next_block, + device=torch.device("cpu"), + dp_group=None, + dp_size=4, + rank=1, + pin_memory=False, + ) + + hook.initialize_hook(block) + + shard = hook.cpu_shards[torch.float32] + # 8 elements / 4 ranks = 2 elements per shard + assert shard.numel() == 2 + # Rank 1 should have elements [102, 103] + assert torch.equal(shard, torch.tensor([102.0, 103.0])) + + def test_sharding_with_remainder(self, dist_group, patched_offload_runtime): + """Verify sharding handles non-even division with padding.""" + block = nn.Module() + block.weight = nn.Parameter(torch.arange(3, dtype=torch.float32)) + + next_block = nn.Module() + next_block.weight = nn.Parameter(torch.arange(100, 103, dtype=torch.float32)) + + # 3 elements, dp_size=2: both ranks get ceil(3/2)=2 elements (padded) + hook0 = DistributedLayerwiseOffloadHook( + next_block=next_block, + device=torch.device("cpu"), + dp_group=None, + dp_size=2, + rank=0, + pin_memory=False, + ) + # Need fresh next_block for each hook since init replaces tensors + next_block0 = nn.Module() + next_block0.weight = nn.Parameter(torch.arange(100, 103, dtype=torch.float32)) + hook0.next_block = next_block0 + hook0.initialize_hook(block) + + shard0 = hook0.cpu_shards[torch.float32] + assert shard0.numel() == 2 + assert torch.equal(shard0, torch.tensor([100.0, 101.0])) + + next_block1 = nn.Module() + next_block1.weight = nn.Parameter(torch.arange(100, 103, dtype=torch.float32)) + block1 = nn.Module() + block1.weight = nn.Parameter(torch.arange(3, dtype=torch.float32)) + hook1 = DistributedLayerwiseOffloadHook( + next_block=next_block1, + device=torch.device("cpu"), + dp_group=None, + dp_size=2, + rank=1, + pin_memory=False, + ) + hook1.initialize_hook(block1) + + shard1 = hook1.cpu_shards[torch.float32] + # Equal-sized shards: rank 1 gets [102, 0] (zero-padded) + assert shard1.numel() == 2 + assert torch.equal(shard1, torch.tensor([102.0, 0.0])) + + +class _DummyBlock(nn.Module): + def __init__(self): + super().__init__() + self.weight = nn.Parameter(torch.randn(10, 10)) + + +class _SingleBlockModel(nn.Module): + _layerwise_offload_blocks_attrs = ["blocks"] + + def __init__(self, num_blocks: int = 3): + super().__init__() + self.blocks = nn.ModuleList([_DummyBlock() for _ in range(num_blocks)]) + + +class _MultiBlockModel(nn.Module): + _layerwise_offload_blocks_attrs = ["transformer_blocks", "single_transformer_blocks"] + + def __init__(self, num_transformer: int = 2, num_single: int = 2): + super().__init__() + self.transformer_blocks = nn.ModuleList([_DummyBlock() for _ in range(num_transformer)]) + self.single_transformer_blocks = nn.ModuleList([_DummyBlock() for _ in range(num_single)]) + + +class _EmptyBlocksModel(nn.Module): + _layerwise_offload_blocks_attrs = ["blocks"] + + def __init__(self): + super().__init__() + self.blocks = nn.ModuleList([]) + + +class _NoAttrsModel(nn.Module): + def __init__(self, num_blocks: int = 2): + super().__init__() + self.blocks = nn.ModuleList([_DummyBlock() for _ in range(num_blocks)]) + + +class _MmapPostLoadModel(nn.Module): + _layerwise_offload_blocks_attrs = ["blocks"] + + def __init__(self): + super().__init__() + self.time_embedder = nn.Linear(2, 2, bias=False) + self.blocks = nn.ModuleList([nn.Linear(2, 2, bias=False) for _ in range(2)]) + self.post_load_calls = 0 + + def post_load_weights(self) -> None: + self.time_embedder.to(torch.float32) + self.post_load_calls += 1 + + +class _MmapPostLoadPipeline(nn.Module): + def __init__(self): + super().__init__() + with torch.device("meta"): + self.transformer = _MmapPostLoadModel() + + @staticmethod + def _remap_ckpt_key(key: str) -> str: + return key + + +class TestMmapWeightLoading: + def test_runs_model_post_load_hook(self, tmp_path, patched_offload_runtime): + pipeline = _MmapPostLoadPipeline() + weights = {name: torch.ones(param.shape, dtype=torch.bfloat16) for name, param in pipeline.named_parameters()} + weight_file = tmp_path / "model.safetensors" + save_file(weights, str(weight_file)) + weight_map = {name: weight_file.name for name in weights} + (tmp_path / "model.safetensors.index.json").write_text(json.dumps({"weight_map": weight_map})) + + backend = DistributedLayerwiseOffloadBackend( + OffloadConfig( + strategy=OffloadStrategy.DISTRIBUTED_LAYER_WISE, + pin_cpu_memory=False, + model_path=str(tmp_path), + ), + torch.device("cpu"), + ) + modules = SimpleNamespace( + dits=[pipeline.transformer], + dit_names=["transformer"], + ) + + backend._load_weights_via_mmap(pipeline, modules) + + assert pipeline.transformer.post_load_calls == 1 + assert pipeline.transformer.time_embedder.weight.dtype == torch.float32 + assert pipeline.transformer.blocks[0].weight.dtype == torch.bfloat16 + + +class TestGetBlocksFromDit: + def test_get_blocks_from_dit_single_block_attr(self): + model = _SingleBlockModel(num_blocks=3) + attr_names, blocks = get_blocks_from_dit(model) + assert attr_names == ["blocks"] + assert len(blocks) == 3 + assert all(isinstance(b, _DummyBlock) for b in blocks) + + def test_get_blocks_from_dit_multi_block_attrs(self): + model = _MultiBlockModel(num_transformer=2, num_single=3) + attr_names, blocks = get_blocks_from_dit(model) + assert set(attr_names) == {"transformer_blocks", "single_transformer_blocks"} + assert len(blocks) == 5 + + def test_get_blocks_from_dit_empty_blocks(self): + model = _EmptyBlocksModel() + attr_names, blocks = get_blocks_from_dit(model) + assert attr_names == [] + assert blocks == [] + + def test_get_blocks_from_dit_no_attrs_defined(self): + model = _NoAttrsModel(num_blocks=3) + attr_names, blocks = get_blocks_from_dit(model) + assert attr_names == [] + assert blocks == [] + + +class TestGetBlocksAttrNames: + def test_get_blocks_attr_names_new_format(self): + model = _MultiBlockModel() + attrs = get_blocks_attr_names(model) + assert attrs == ["transformer_blocks", "single_transformer_blocks"] + + def test_get_blocks_attr_names_no_attrs(self): + model = _NoAttrsModel() + attrs = get_blocks_attr_names(model) + assert attrs == [] + + def test_set_blocks_attr_names(self): + model = _NoAttrsModel() + set_blocks_attr_names(model, ["new_blocks"]) + assert hasattr(model.__class__, "_layerwise_offload_blocks_attrs") + assert model.__class__._layerwise_offload_blocks_attrs == ["new_blocks"] + + +class TestCrossGroupSharedBuffer: + """Regression test: when multiple DiT groups share the same 2 GPU + buffers, the first block of each group must always sync-prefetch on + entry, because another group may have overwritten the shared slot. + """ + + def test_group_first_hook_forces_sync_prefetch(self, dist_group, patched_offload_runtime): + """Verify _is_group_first=True forces sync-prefetch in pre_forward + even when is_materialized returns True.""" + block_a = TinyBlock(_make_values(1.0)) + block_b = TinyBlock(_make_values(10.0)) + + hook_a = DistributedLayerwiseOffloadHook( + next_block=block_b, + device=torch.device("cpu"), + dp_group=None, + dp_size=1, + rank=0, + pin_memory=False, + ) + hook_a.initialize_hook(block_a) + + # Prefetch to make block_a "materialized" (non-empty params) + hook_a.prefetch_layer(hook_a.current_slot, non_blocking=False) + assert hook_a.is_materialized + + # Track sync-prefetch calls + sync_called = [False] + orig = hook_a.prefetch_layer + + def tracking(slot, non_blocking=True): + if not non_blocking: + sync_called[0] = True + orig(slot, non_blocking=non_blocking) + + hook_a.prefetch_layer = tracking + + # Case 1: _is_group_first=False, materialized → no sync-prefetch + hook_a._is_group_first = False + sync_called[0] = False + hook_a.pre_forward(block_a) + assert not sync_called[0], "Materialized block should not sync-prefetch without _is_group_first" + + # Case 2: _is_group_first=True, materialized, slot contaminated + # (no _shared_slot_group set → defaults to contaminated) → MUST sync-prefetch + hook_a._is_group_first = True + hook_a._prev_hook = hook_a + sync_called[0] = False + hook_a.pre_forward(block_a) + assert sync_called[0], "_is_group_first must force sync-prefetch when slot is contaminated" + + # Case 3: _is_group_first=True, slot NOT contaminated (same group owns slot) + # → skip sync-prefetch, just wait for async event + hook_a._shared_slot_group = [hook_a._group_id, -1] + sync_called[0] = False + hook_a.pre_forward(block_a) + assert not sync_called[0], "Should skip sync-prefetch when slot_group matches (non-contaminated)" + + +class TestOffloadPlan: + """Test declarative OffloadPlan metadata.""" + + def test_get_offload_plan_returns_none_when_not_declared(self): + """Pipelines without _offload_plan should return None.""" + model = _SingleBlockModel(num_blocks=3) + assert get_offload_plan(model) is None + + def test_get_offload_plan_returns_declared_plan(self): + """Pipelines with _offload_plan should return it.""" + plan = OffloadPlan( + block_attrs={"transformer": ("gen_layers",)}, + offload_submodules={"context_encoder": "layers"}, + ) + + class PipelineWithPlan(nn.Module): + _offload_plan = plan + + model = PipelineWithPlan() + result = get_offload_plan(model) + assert result is plan + assert result.block_attrs == {"transformer": ("gen_layers",)} + assert result.offload_submodules == {"context_encoder": "layers"} + + def test_offload_plan_defaults_to_empty(self): + """OffloadPlan with no arguments should have empty dicts.""" + plan = OffloadPlan() + assert plan.block_attrs == {} + assert plan.offload_submodules == {} + + def test_offload_plan_is_frozen(self): + """OffloadPlan should be immutable (frozen=True).""" + plan = OffloadPlan() + with pytest.raises(Exception): + plan.block_attrs = {"x": ("y",)} # type: ignore + + +class TestMmapValidation: + """Tests for mmap loader validation: TP rejection, strict check, validate_loaded_weights.""" + + def test_tp_rejected_when_tp_world_size_gt_1(self, tmp_path, patched_offload_runtime, monkeypatch): + """DLO+AllGather should reject when TP world size > 1.""" + import json + from types import SimpleNamespace + + from safetensors.torch import save_file + + from vllm_omni.diffusion.offloader.base import OffloadConfig, OffloadStrategy + + # Mock TP world size = 2 + monkeypatch.setattr( + "vllm.distributed.parallel_state.get_tensor_model_parallel_world_size", + lambda: 2, + ) + + class SimpleModel(nn.Module): + _layerwise_offload_blocks_attrs = ["blocks"] + + def __init__(self): + super().__init__() + self.blocks = nn.ModuleList([nn.Linear(4, 4, bias=False)]) + + class SimplePipeline(nn.Module): + def __init__(self): + super().__init__() + self.transformer = SimpleModel() + + @staticmethod + def _remap_ckpt_key(key): + return key + + pipeline = SimplePipeline() + weights = {name: torch.ones(p.shape, dtype=p.dtype) for name, p in pipeline.named_parameters() if not p.is_meta} + save_file(weights, str(tmp_path / "model.safetensors")) + (tmp_path / "model.safetensors.index.json").write_text( + json.dumps({"weight_map": {k: "model.safetensors" for k in weights}}) + ) + + backend = DistributedLayerwiseOffloadBackend( + OffloadConfig( + strategy=OffloadStrategy.DISTRIBUTED_LAYER_WISE, + pin_cpu_memory=False, + model_path=str(tmp_path), + ), + torch.device("cpu"), + ) + modules = SimpleNamespace(dits=[pipeline.transformer], dit_names=["transformer"]) + + with pytest.raises(ValueError, match="Tensor Parallel"): + backend._load_weights_via_mmap(pipeline, modules) + + def test_tp_allowed_when_tp_world_size_eq_1(self, tmp_path, patched_offload_runtime, monkeypatch): + """DLO+AllGather should work when TP world size = 1, even if params + have custom weight_loader attributes (e.g. QKVParallelLinear at TP=1).""" + import json + from types import SimpleNamespace + + from safetensors.torch import save_file + + from vllm_omni.diffusion.offloader.base import OffloadConfig, OffloadStrategy + + # Mock TP world size = 1 (default) + monkeypatch.setattr( + "vllm.distributed.parallel_state.get_tensor_model_parallel_world_size", + lambda: 1, + ) + + def custom_weight_loader(param, weight): + param.data.copy_(weight) + + class ModelWithCustomLoader(nn.Module): + _layerwise_offload_blocks_attrs = ["blocks"] + + def __init__(self): + super().__init__() + self.linear = nn.Linear(4, 4, bias=False) + # Simulate QKVParallelLinear: custom weight_loader even at TP=1 + self.linear.weight.weight_loader = custom_weight_loader + self.blocks = nn.ModuleList([nn.Linear(4, 4, bias=False)]) + + class PipelineWithLoader(nn.Module): + def __init__(self): + super().__init__() + self.transformer = ModelWithCustomLoader() + + @staticmethod + def _remap_ckpt_key(key): + return key + + pipeline = PipelineWithLoader() + weights = {name: torch.ones(p.shape, dtype=p.dtype) for name, p in pipeline.named_parameters() if not p.is_meta} + save_file(weights, str(tmp_path / "model.safetensors")) + (tmp_path / "model.safetensors.index.json").write_text( + json.dumps({"weight_map": {k: "model.safetensors" for k in weights}}) + ) + + backend = DistributedLayerwiseOffloadBackend( + OffloadConfig( + strategy=OffloadStrategy.DISTRIBUTED_LAYER_WISE, + pin_cpu_memory=False, + model_path=str(tmp_path), + ), + torch.device("cpu"), + ) + modules = SimpleNamespace(dits=[pipeline.transformer], dit_names=["transformer"]) + + # Should NOT raise — TP=1 is allowed even with custom weight_loader + backend._load_weights_via_mmap(pipeline, modules) + + def test_validate_loaded_weights_called(self, tmp_path, patched_offload_runtime): + """validate_loaded_weights should be called after mmap loading.""" + import json + from types import SimpleNamespace + + from safetensors.torch import save_file + + from vllm_omni.diffusion.offloader.base import OffloadConfig, OffloadStrategy + + class ModelWithValidate(nn.Module): + _layerwise_offload_blocks_attrs = ["blocks"] + validate_called = False + + def __init__(self): + super().__init__() + self.blocks = nn.ModuleList([nn.Linear(4, 4, bias=False) for _ in range(2)]) + + def post_load_weights(self): + pass + + def validate_loaded_weights(self, loaded): + self.validate_called = True + assert len(loaded) > 0, "Should have loaded weights" + + class PipelineWithValidate(nn.Module): + def __init__(self): + super().__init__() + self.transformer = ModelWithValidate() + + @staticmethod + def _remap_ckpt_key(key): + return key + + pipeline = PipelineWithValidate() + # Save on meta to trigger mmap path + with torch.device("meta"): + pipeline.transformer = ModelWithValidate() + + weights = { + name: torch.ones(param.shape, dtype=param.dtype if not param.is_meta else torch.float32) + for name, param in pipeline.named_parameters() + } + save_file(weights, str(tmp_path / "model.safetensors")) + (tmp_path / "model.safetensors.index.json").write_text( + json.dumps({"weight_map": {k: "model.safetensors" for k in weights}}) + ) + + backend = DistributedLayerwiseOffloadBackend( + OffloadConfig( + strategy=OffloadStrategy.DISTRIBUTED_LAYER_WISE, pin_cpu_memory=False, model_path=str(tmp_path) + ), + torch.device("cpu"), + ) + modules = SimpleNamespace(dits=[pipeline.transformer], dit_names=["transformer"]) + + backend._load_weights_via_mmap(pipeline, modules) + assert pipeline.transformer.validate_called, "validate_loaded_weights should be called" + + +class TestConfigValidation: + """Tests for configuration validation in OffloadConfig / execute_request.""" + + def test_hsdp_with_allgather_rejected(self): + """HSDP + DLO + AllGather should raise ValueError (double sharding).""" + from vllm_omni.diffusion.offloader.base import OffloadConfig + + class FakePC: + data_parallel_size = 1 + use_hsdp = True + hsdp_shard_size = 2 + hsdp_replicate_size = 1 + sequence_parallel_size = 1 + tensor_parallel_size = 1 + cfg_parallel_size = 1 + pipeline_parallel_size = 1 + ulysses_degree = 1 + ring_degree = 1 + allgather_degree = 1 + enable_expert_parallel = False + + class FakeODConfig: + enable_cpu_offload = False + enable_layerwise_offload = False + enable_distributed_layerwise_offload = True + dlo_use_allgather = True + pin_cpu_memory = True + parallel_config = FakePC() + model = "/fake/path" + + with pytest.raises(ValueError, match="incompatible with HSDP"): + OffloadConfig.from_od_config(FakeODConfig()) + + def test_hsdp_without_allgather_allowed(self): + """HSDP + DLO + no-AllGather should be allowed (full weights per rank).""" + from vllm_omni.diffusion.offloader.base import OffloadConfig + + class FakePC: + data_parallel_size = 1 + use_hsdp = True + hsdp_shard_size = 2 + hsdp_replicate_size = 1 + sequence_parallel_size = 1 + tensor_parallel_size = 1 + cfg_parallel_size = 1 + pipeline_parallel_size = 1 + ulysses_degree = 1 + ring_degree = 1 + allgather_degree = 1 + enable_expert_parallel = False + + class FakeODConfig: + enable_cpu_offload = False + enable_layerwise_offload = False + enable_distributed_layerwise_offload = True + dlo_use_allgather = False # no AllGather → should be allowed + pin_cpu_memory = True + parallel_config = FakePC() + model = "/fake/path" + + config = OffloadConfig.from_od_config(FakeODConfig()) + assert config.dp_size == 1 # forced to 1 when no AllGather + + def test_num_inference_steps_none_rejected(self): + """DP multi-concurrency should reject None num_inference_steps.""" + from types import SimpleNamespace + + # Mock requests with None steps + reqs = [ + SimpleNamespace( + req=SimpleNamespace( + request_id=f"req-{i}", + sampling_params=SimpleNamespace(num_inference_steps=None), + ) + ) + for i in range(2) + ] + + # We can't easily instantiate the full executor, but we can test + # the validation logic by checking that the code path raises. + # The validation is in execute_request, which needs self._ensure_open(). + # Instead, test the validation logic directly: + step_counts = { + r.req.sampling_params.num_inference_steps + for r in reqs + if r.req.sampling_params.num_inference_steps is not None + } + has_none = any(r.req.sampling_params.num_inference_steps is None for r in reqs) + assert has_none, "Test setup: should have None steps" + assert len(step_counts) == 0, "Test setup: no explicit steps" + + # The validation condition: (len(step_counts) > 1) or has_none → should reject + should_reject = (len(step_counts) > 1) or has_none + assert should_reject, "None steps should trigger rejection" + + def test_num_inference_steps_same_explicit_allowed(self): + """DP multi-concurrency should allow same explicit num_inference_steps.""" + from types import SimpleNamespace + + reqs = [ + SimpleNamespace( + req=SimpleNamespace( + request_id=f"req-{i}", + sampling_params=SimpleNamespace(num_inference_steps=35), + ) + ) + for i in range(4) + ] + + step_counts = { + r.req.sampling_params.num_inference_steps + for r in reqs + if r.req.sampling_params.num_inference_steps is not None + } + has_none = any(r.req.sampling_params.num_inference_steps is None for r in reqs) + + should_reject = (len(step_counts) > 1) or has_none + assert not should_reject, "Same explicit steps should be allowed" + assert step_counts == {35} + + def test_num_inference_steps_different_explicit_rejected(self): + """DP multi-concurrency should reject different explicit steps.""" + from types import SimpleNamespace + + reqs = [ + SimpleNamespace( + req=SimpleNamespace( + request_id="req-0", + sampling_params=SimpleNamespace(num_inference_steps=35), + ) + ), + SimpleNamespace( + req=SimpleNamespace( + request_id="req-1", + sampling_params=SimpleNamespace(num_inference_steps=30), + ) + ), + ] + + step_counts = { + r.req.sampling_params.num_inference_steps + for r in reqs + if r.req.sampling_params.num_inference_steps is not None + } + has_none = any(r.req.sampling_params.num_inference_steps is None for r in reqs) + + should_reject = (len(step_counts) > 1) or has_none + assert should_reject, "Different steps should trigger rejection" + assert len(step_counts) == 2 + + +class TestDynamicSlotTracking: + """Tests for dynamic slot tracking with odd and even block counts. + + With i%2 initialization alone, odd block counts cause the circular + tail→head prefetch to collide with the head's read slot (both map + to slot 0). Dynamic slot tracking via _prefetched_slot corrects + this by reading the actual slot from the previous hook's prefetch. + """ + + @staticmethod + def _trace_slots(num_blocks: int, num_iterations: int = 1): + """Trace slot assignments through forward passes (pure logic, no hooks). + + Returns a list of (read_slot, write_slot) per block per iteration. + """ + # Initial i%2 assignment (same as enable()) + current_slots = [i % 2 for i in range(num_blocks)] + prefetched_slots: list[int | None] = [None] * num_blocks + + # Sync-prefetch: tail hook prefetches into slot 0 + prefetched_slots[-1] = 0 + + results = [] + for _ in range(num_iterations): + for i in range(num_blocks): + prev_idx = (i - 1) % num_blocks + + # Dynamic slot tracking: read from prev's prefetched slot + if prefetched_slots[prev_idx] is not None: + current_slots[i] = prefetched_slots[prev_idx] + + read_slot = current_slots[i] + + # Prefetch into other slot + next_slot = 1 - read_slot + current_slots[i] = next_slot + prefetched_slots[i] = next_slot + + results.append((read_slot, next_slot)) + + return results + + def test_odd_block_count_no_collision(self): + """3 blocks (odd): no read/write slot collision.""" + results = self._trace_slots(3) + for i, (read, write) in enumerate(results): + assert read != write, f"Block {i % 3} (step {i}): read={read} == write={write}" + + def test_even_block_count_no_collision(self): + """4 blocks (even): no read/write slot collision.""" + results = self._trace_slots(4) + for i, (read, write) in enumerate(results): + assert read != write, f"Block {i % 4} (step {i}): read={read} == write={write}" + + def test_large_odd_block_count(self): + """7 blocks (odd): no collision across all blocks.""" + results = self._trace_slots(7) + for i, (read, write) in enumerate(results): + assert read != write, f"Block {i % 7} (step {i}): read={read} == write={write}" + + def test_5_blocks_two_iterations(self): + """5 blocks (odd), 2 iterations: no collision in either pass.""" + results = self._trace_slots(5, num_iterations=2) + n = 5 + for iteration in range(2): + for i in range(n): + idx = iteration * n + i + read, write = results[idx] + assert read != write, f"Iter {iteration} block {i}: read={read} == write={write}" + + def test_3_blocks_three_iterations(self): + """3 blocks (odd), 3 iterations: no collision across all passes.""" + results = self._trace_slots(3, num_iterations=3) + n = 3 + for iteration in range(3): + for i in range(n): + idx = iteration * n + i + read, write = results[idx] + assert read != write, f"Iter {iteration} block {i}: read={read} == write={write}" + + def test_72_blocks_cosmos3(self): + """72 blocks (Cosmos3, even): no collision.""" + results = self._trace_slots(72) + for i, (read, write) in enumerate(results): + assert read != write, f"Block {i % 72} (step {i}): read={read} == write={write}" + + def test_slots_alternate_within_iteration(self): + """Within one iteration, consecutive blocks should use alternating slots.""" + results = self._trace_slots(6) + for i in range(6): + read, write = results[i] + # Each block reads from one slot and writes to the other + assert {read, write} == {0, 1}, f"Block {i}: slots {read},{write} don't cover both" + + def test_prefetched_slot_set_after_prefetch(self, dist_group, patched_offload_runtime): + """prefetch_layer should set _prefetched_slot to the slot argument.""" + block = TinyBlock(_make_values(1.0)) + next_block = TinyBlock(_make_values(2.0)) + + hook = DistributedLayerwiseOffloadHook( + next_block=next_block, + device=torch.device("cpu"), + dp_group=None, + dp_size=1, + rank=0, + copy_stream=DummyStream(), + comm_stream=DummyStream(), + pin_memory=False, + ) + hook.initialize_hook(block) + + assert hook._prefetched_slot is None # initially None + + hook.prefetch_layer(slot=0, non_blocking=False) + assert hook._prefetched_slot == 0 + + hook.prefetch_layer(slot=1, non_blocking=False) + assert hook._prefetched_slot == 1 + + def test_dynamic_slot_overrides_initial_assignment(self, dist_group, patched_offload_runtime): + """When _prev_hook._prefetched_slot is set, pre_forward should use + that value instead of the initial i%2 assignment.""" + block_a = TinyBlock(_make_values(1.0)) + block_b = TinyBlock(_make_values(2.0)) + + hook_a = DistributedLayerwiseOffloadHook( + next_block=block_b, + device=torch.device("cpu"), + dp_group=None, + dp_size=1, + rank=0, + copy_stream=DummyStream(), + comm_stream=DummyStream(), + pin_memory=False, + ) + hook_a.initialize_hook(block_a) + + hook_b = DistributedLayerwiseOffloadHook( + next_block=block_a, + device=torch.device("cpu"), + dp_group=None, + dp_size=1, + rank=0, + copy_stream=DummyStream(), + comm_stream=DummyStream(), + pin_memory=False, + ) + hook_b.initialize_hook(block_b) + + # Wire: hook_b._prev_hook = hook_a + hook_b._prev_hook = hook_a + + # hook_a prefetched into slot 1 + hook_a.prefetch_layer(slot=1, non_blocking=False) + assert hook_a._prefetched_slot == 1 + + # hook_b's initial slot is 0 (from i%2) + hook_b.current_slot = 0 + + # pre_forward should override to slot 1 + hook_b.pre_forward(block_b) + assert hook_b.current_slot == 1, "pre_forward should read _prev_hook._prefetched_slot=1, not keep initial 0" diff --git a/tests/diffusion/test_diffusion_engine.py b/tests/diffusion/test_diffusion_engine.py index 192675908..e26211e9c 100644 --- a/tests/diffusion/test_diffusion_engine.py +++ b/tests/diffusion/test_diffusion_engine.py @@ -92,6 +92,21 @@ class MockScheduler: # assume all new req finished return [req.request_id for req in sched_output.scheduled_new_reqs] + def get_request_state(self, request_id): + return None + + def pop_request_state(self, request_id): + return None + + def finish_requests(self, request_id, status): + pass + + def close(self): + pass + + def initialize(self, od_config): + pass + class _BatchCapablePipeline: supports_request_batch = True @@ -644,7 +659,7 @@ async def test_async_add_req_and_stream_response(): engine.step_execution = False engine.execution_mode = DiffusionExecutionMode.REQUEST_BATCH - def _finalize(rid, out, err=None): + def _finalize(rid, out, err=None, **kwargs): # Stream consumers stop on ``finished``; keep result_data for assertions. return SimpleNamespace(result_data=out.result.result_data, finished=True) diff --git a/tests/diffusion/test_multiproc_engine_concurrency.py b/tests/diffusion/test_multiproc_engine_concurrency.py index 10fe7e86c..932630a23 100644 --- a/tests/diffusion/test_multiproc_engine_concurrency.py +++ b/tests/diffusion/test_multiproc_engine_concurrency.py @@ -102,7 +102,10 @@ def _start_worker(req_q, res_q, count=2): def _run(): for _ in range(count): - req = req_q.get(timeout=10) + try: + req = req_q.get(timeout=10) + except queue.Empty: + break method = req.get("method", "") args = req.get("args", ()) if method == "execute_model_batch" and args and isinstance(args[0], DiffusionSchedulerOutput): @@ -146,7 +149,7 @@ def _inject_interleave(executor): orig_enqueue(item) if threading.current_thread().name == "thread_a": a_enqueued.set() # tell B: "A has enqueued" - b_complete.wait(5) # block A until B finishes + b_complete.wait(1) # block A until B finishes executor._broadcast_mq.enqueue = _controlled return a_enqueued, b_complete @@ -173,8 +176,8 @@ class TestConcurrentRequestExecution: results["B"] = engine.add_req_and_wait_for_response(_mock_request("B")) b_complete.set() # release A - ta = threading.Thread(target=_a, name="thread_a") - tb = threading.Thread(target=_b, name="thread_b") + ta = threading.Thread(target=_a, name="thread_a", daemon=True) + tb = threading.Thread(target=_b, name="thread_b", daemon=True) ta.start() tb.start() ta.join(10) @@ -274,8 +277,8 @@ class TestConcurrentCollectiveRpc: ) b_complete.set() - ta = threading.Thread(target=_a, name="thread_a") - tb = threading.Thread(target=_b, name="thread_b") + ta = threading.Thread(target=_a, name="thread_a", daemon=True) + tb = threading.Thread(target=_b, name="thread_b", daemon=True) ta.start() tb.start() ta.join(10) @@ -311,8 +314,8 @@ class TestConcurrentRequestExecutionAndCollectiveRpc: ) b_complete.set() - ta = threading.Thread(target=_a, name="thread_a") - tb = threading.Thread(target=_b, name="thread_b") + ta = threading.Thread(target=_a, name="thread_a", daemon=True) + tb = threading.Thread(target=_b, name="thread_b", daemon=True) ta.start() tb.start() ta.join(10) @@ -683,7 +686,10 @@ class TestMultiprocExecutorStepStreamingOutput: def _worker(): for runner_output in runner_outputs: - req_q.get(timeout=10) + try: + req_q.get(timeout=10) + except queue.Empty: + break res_q.put(runner_output) thread = threading.Thread(target=_worker, daemon=True) diff --git a/vllm_omni/config/omni_config.py b/vllm_omni/config/omni_config.py index 422e8d784..c8fa51448 100644 --- a/vllm_omni/config/omni_config.py +++ b/vllm_omni/config/omni_config.py @@ -474,6 +474,8 @@ class _DiffusionConfigProjection: output_type: str = "pil" enable_cpu_offload: bool = False enable_layerwise_offload: bool = False + enable_distributed_layerwise_offload: bool = False + dlo_use_allgather: bool = True pin_cpu_memory: bool = True diffusion_compile_granularity: Literal["regional", "full"] = "regional" diffusion_compile_dynamic: bool = Field(default=True, strict=True) diff --git a/vllm_omni/diffusion/data.py b/vllm_omni/diffusion/data.py index b97ffc311..31617ccdb 100644 --- a/vllm_omni/diffusion/data.py +++ b/vllm_omni/diffusion/data.py @@ -672,6 +672,12 @@ class OmniDiffusionConfig: enable_cpu_offload: bool = False # Layer-wise offloading (block-level offloading) parameters enable_layerwise_offload: bool = False + # Distributed layer-wise offloading with H2D + AllGather overlap (RFC-1) + enable_distributed_layerwise_offload: bool = False + # If True: shard weights 1/dp_size + AllGather (saves CPU memory, requires + # concurrent requests in DP mode). If False: each rank loads full weights + # via H2D only (N× CPU memory, but no AllGather synchronization needed). + dlo_use_allgather: bool = True pin_cpu_memory: bool = True # Use pinned memory for faster transfers when offloading diff --git a/vllm_omni/diffusion/diffusion_engine.py b/vllm_omni/diffusion/diffusion_engine.py index 44ca9da89..e89b95571 100644 --- a/vllm_omni/diffusion/diffusion_engine.py +++ b/vllm_omni/diffusion/diffusion_engine.py @@ -156,6 +156,10 @@ class DiffusionEngine: #: remains a pure explicit user override (never mutated by engines). default_diffusion_model_runner_cls: str | None = None + # Class-level default so tests using object.__new__ (without __init__) + # don't hit AttributeError when _busy_loop accesses self.dp_concurrent. + dp_concurrent: bool = False + def __init__( self, od_config: OmniDiffusionConfig, @@ -224,6 +228,29 @@ class DiffusionEngine: self.scheduler.initialize(od_config) def _init_runtime_state(self) -> None: + # DP multi-concurrency: allow batching dp_size requests so each + # worker processes a different request in parallel. Only enabled + # for distributed layerwise offload (which shards weights and + # needs all ranks active simultaneously). Ordinary DP with a + # non-batch pipeline should not schedule multiple requests. + dp_size = 1 + if getattr(self.od_config, "parallel_config", None) is not None: + dp_size = getattr(self.od_config.parallel_config, "data_parallel_size", 1) + dist_offload = getattr(self.od_config, "enable_distributed_layerwise_offload", False) + if dp_size > 1 and dist_offload: + self.scheduler.max_num_running_reqs = dp_size + self.dp_concurrent = True + # Set batch admission wait so concurrent requests accumulate + # before scheduling. The scheduler reads this from od_config. + if getattr(self.od_config, "request_batch_max_wait_ms", 0) == 0: + self.od_config.request_batch_max_wait_ms = 500.0 + logger.info( + "dp_concurrent: max_num_running_reqs=%d, batch_wait=%sms", + dp_size, + self.od_config.request_batch_max_wait_ms, + ) + else: + self.dp_concurrent = False self.main_loop: asyncio.AbstractEventLoop | None = None self.stop_event: threading.Event | None = None self.worker_thread: threading.Thread | None = None @@ -402,7 +429,7 @@ class DiffusionEngine: # Only RPC / abort work pending; loop back to drain it. continue - if self.execution_mode == DiffusionExecutionMode.REQUEST_BATCH: + if self.supports_request_batch or self.dp_concurrent: self._wait_for_request_batch_admission_locked() sched_output = self.scheduler.schedule() @@ -442,7 +469,7 @@ class DiffusionEngine: Caller must hold ``self._cv``. """ - if self.step_execution or not self.supports_request_batch: + if self.step_execution or (not self.supports_request_batch and not self.dp_concurrent): return max_wait_s = self.od_config.request_batch_max_wait_ms / 1000.0 @@ -460,9 +487,12 @@ class DiffusionEngine: deadline = start + max_wait_s last_waiting = -1 stable_since = start - # Require a short idle period with no queue growth so bursty HTTP - # ingress can land before the first schedule() of a wave. - stable_window_s = min(0.05, max_wait_s / 5.0) + # For dp_concurrent, use a longer stable window so all dp_size + # HTTP requests have time to land before scheduling. + if self.dp_concurrent: + stable_window_s = min(0.3, max_wait_s / 2.0) + else: + stable_window_s = min(0.05, max_wait_s / 5.0) while not self.stop_event.is_set(): waiting = self.scheduler.num_waiting_requests() @@ -783,6 +813,27 @@ class DiffusionEngine: raise RuntimeError(f"Could not {action} profiler: {e}") from e def run_startup_warmup(self) -> None: + dlo_use_allgather = getattr(self.od_config, "dlo_use_allgather", True) + # Skip dummy run when AllGather is used with more than 1 rank, + # because the dummy run sends only 1 request but AllGather requires + # all ranks to participate simultaneously. This covers both DP > 1 + # and SP > 1 (where dp_size is derived from sp_size in OffloadConfig). + pc = getattr(self.od_config, "parallel_config", None) + dp_size = getattr(pc, "data_parallel_size", 1) if pc else 1 + sp_size = getattr(pc, "sequence_parallel_size", 1) if pc else 1 + effective_shard_size = max(dp_size, sp_size) + skip_dummy = ( + getattr(self.od_config, "enable_distributed_layerwise_offload", False) + and dlo_use_allgather + and effective_shard_size > 1 + ) + if skip_dummy: + logger.info( + "Skipping dummy run (dist_offload with AllGather, dp_size=%d, sp_size=%d)", + dp_size, + sp_size, + ) + return try: self._dummy_run() except Exception as e: diff --git a/vllm_omni/diffusion/executor/multiproc_executor.py b/vllm_omni/diffusion/executor/multiproc_executor.py index 84a460c53..fa0213fca 100644 --- a/vllm_omni/diffusion/executor/multiproc_executor.py +++ b/vllm_omni/diffusion/executor/multiproc_executor.py @@ -1,6 +1,7 @@ from __future__ import annotations import concurrent.futures +import json import multiprocessing as mp import multiprocessing.connection import queue @@ -64,12 +65,17 @@ class _ExecutorShutdownCleaner: class MultiprocDiffusionExecutor(DiffusionExecutor): uses_multiproc: bool = True + # Class-level defaults so tests using object.__new__ (without _init_executor) + # don't hit AttributeError when collective_rpc accesses these. + _rpc_wave_id: int = 0 + def _init_executor(self) -> None: self._processes: list[mp.Process] = [] self._closed = False self._is_failed = False self._failure_callbacks: list[Callable[[], None]] = [] self._result_mq: MessageQueue | None = None + self._rpc_wave_id: int = 0 num_workers = cast(int, self.od_config.num_gpus) self.wake_events = [mp.Event() for _ in range(num_workers)] @@ -215,6 +221,30 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): MultiprocDiffusionExecutor._raise_for_rpc_error_dict(response) return response + _MAX_STALE_DISCARDS = 16 + + def _validate_wave_id(self, response: Any, expected_wave_id: int, deadline: float | None, method: str) -> Any: + """Discard stale RPC responses from a previous wave.""" + discards = 0 + while discards < self._MAX_STALE_DISCARDS: + if not isinstance(response, dict): + return response + resp_wave_id = response.get("wave_id") + if resp_wave_id is None or resp_wave_id == expected_wave_id: + return response + logger.warning( + "Discarding stale RPC response (wave_id=%s, expected=%s, method=%s)", + resp_wave_id, + expected_wave_id, + method, + ) + discards += 1 + response = self._dequeue_one_with_failure_polling(deadline, method) + raise TimeoutError( + f"Discarded {self._MAX_STALE_DISCARDS} stale RPC responses " + f"without finding wave_id={expected_wave_id} for method={method}." + ) + def _launch_workers( self, broadcast_handle: Handle, @@ -355,9 +385,84 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): from vllm_omni.diffusion.worker.utils import BatchRunnerOutput, RunnerOutput self._ensure_open() + new_reqs = scheduler_output.scheduled_new_reqs runner_outputs: list[RunnerOutput] = [] - for new_req in scheduler_output.scheduled_new_reqs: + # DP multi-concurrency: when DLO+AllGather is active and multiple + # requests are scheduled, send ALL requests in one broadcast RPC. + # Each rank picks req[rank % len(reqs)] and computes independently. + # All ranks reply (unique_reply_rank=None) so we collect dp_size + # responses and match by dp_rank. + if ( + len(new_reqs) > 1 + and getattr(self.od_config, "enable_distributed_layerwise_offload", False) + and getattr(self.od_config, "dlo_use_allgather", True) + ): + # Validate: all concurrent requests must share the same + # num_inference_steps and identical extra_args, because + # AllGather is a collective that requires every rank to + # participate at each step. + step_counts = { + nr.req.sampling_params.num_inference_steps + for nr in new_reqs + if nr.req.sampling_params.num_inference_steps is not None + } + has_none = any(nr.req.sampling_params.num_inference_steps is None for nr in new_reqs) + if (len(step_counts) > 1) or has_none: + raise ValueError( + "DP multi-concurrency requires all concurrent requests to have " + "the same explicit num_inference_steps (None is not allowed), got " + f"{[nr.req.sampling_params.num_inference_steps for nr in new_reqs]}." + ) + extra_args_signatures: set = set() + for nr in new_reqs: + ea = getattr(nr.req, "extra_args", None) + if ea and isinstance(ea, dict): + extra_args_signatures.add(json.dumps(ea, sort_keys=True)) + else: + extra_args_signatures.add(None) + if len(extra_args_signatures) > 1: + raise ValueError( + "DP multi-concurrency requires all concurrent requests to " + "share identical extra_args. Different extra_args can change " + "the forward schedule and cause AllGather deadlock." + ) + + reqs_list = [nr.req for nr in new_reqs] + try: + results = self.collective_rpc( + "execute_model", + args=(reqs_list, self.od_config, scheduler_output.kv_prefetch_job), + unique_reply_rank=None, + exec_all_ranks=True, + ) + results = results if isinstance(results, list) else [results] + for i, new_req in enumerate(new_reqs): + res = results[i] if i < len(results) else results[0] + if isinstance(res, DiffusionOutput): + runner_outputs.append( + RunnerOutput( + request_id=new_req.request_id, + step_index=None, + finished=True, + result=res, + ) + ) + else: + raise RuntimeError(f"Unexpected response type [{i}]: {type(res)!r}") + except Exception as exc: + for new_req in new_reqs: + runner_outputs.append( + RunnerOutput( + request_id=new_req.request_id, + step_index=None, + finished=True, + result=DiffusionOutput(error=str(exc)), + ) + ) + return BatchRunnerOutput.from_list(runner_outputs) + + for new_req in new_reqs: req = new_req.req try: result = self.collective_rpc( @@ -511,24 +616,61 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): # ── Path 2: step-execution or other non-execute_model RPCs ── # _dequeue_one_with_failure_polling reads from _sync_result_buffer # when pump is active (request-mode), or result_mq directly otherwise. + self._rpc_wave_id += 1 + wave_id = self._rpc_wave_id + rpc_request["wave_id"] = wave_id + try: # Broadcast RPC request to all workers via unified message queue self._broadcast_mq.enqueue(rpc_request) # pyright: ignore[reportOptionalMemberAccess] MQ is not None before shutdown - num_responses = 1 + # Determine number of responses to collect: + # - unique_reply_rank=None + exec_all_ranks=True: all DP ranks reply + # (N responses, one per DP worker). + # - Otherwise: 1 response (only rank 0 or specified rank) + if unique_reply_rank is None and exec_all_ranks: + dp_size = getattr(self.od_config.parallel_config, "data_parallel_size", 1) + num_responses = max(1, dp_size) + else: + num_responses = 1 - responses = [] - for _ in range(num_responses): - response = self._dequeue_one_with_failure_polling(deadline, method) + responses: list = [] + if unique_reply_rank is None and exec_all_ranks and num_responses > 1: + # DP multi-concurrency: collect num_responses replies, sort by dp_rank. + tagged: list[tuple[int, Any]] = [] + collected_errors: list[str] = [] + for _ in range(num_responses): + response = self._dequeue_one_with_failure_polling(deadline, method) + response = self._validate_wave_id(response, wave_id, deadline, method) + try: + unpack_diffusion_output_shm(response) + except Exception as e: + logger.warning("SHM unpack failed (data may already be inline): %s", e) + if isinstance(response, dict) and response.get("status") == "error": + collected_errors.append(str(response.get("error", "unknown"))) + else: + response = MultiprocDiffusionExecutor._handle_rpc_response(response) + if isinstance(response, dict) and "dp_rank" in response: + tagged.append((response["dp_rank"], response["output"])) + else: + tagged.append((len(tagged), response)) + if collected_errors: + raise RuntimeError(f"Worker error: {collected_errors[0]}") + tagged.sort(key=lambda x: x[0]) + responses = [r for _, r in tagged] + else: + for _ in range(num_responses): + response = self._dequeue_one_with_failure_polling(deadline, method) + response = self._validate_wave_id(response, wave_id, deadline, method) - try: - unpack_diffusion_output_shm(response) - except Exception as e: - logger.warning("SHM unpack failed (data may already be inline): %s", e) + try: + unpack_diffusion_output_shm(response) + except Exception as e: + logger.warning("SHM unpack failed (data may already be inline): %s", e) - response = MultiprocDiffusionExecutor._handle_rpc_response(response) + response = MultiprocDiffusionExecutor._handle_rpc_response(response) - responses.append(response) + responses.append(response) return responses[0] if unique_reply_rank is not None else responses except Exception as e: diff --git a/vllm_omni/diffusion/ipc.py b/vllm_omni/diffusion/ipc.py index ca6bc858d..5b7465af0 100644 --- a/vllm_omni/diffusion/ipc.py +++ b/vllm_omni/diffusion/ipc.py @@ -172,8 +172,9 @@ def pack_diffusion_output_shm( Supports a bare ``DiffusionOutput``, a wrapper object carrying one in ``.result`` (for example ``RunnerOutput``), an RPC result envelope carrying - the diffusion output in ``["result"]``, or a batch wrapper carrying - ``RunnerOutput`` objects in ``.runner_outputs``. + the diffusion output in ``["result"]``, a batch wrapper carrying + ``RunnerOutput`` objects in ``.runner_outputs``, or a DP-tagged dict + ``{"dp_rank": int, "output": DiffusionOutput}`` used by DP multi-concurrency. If *d2h_stream* is provided, D2H copies use that stream (non-blocking on the default stream). The caller must synchronize *d2h_stream* afterward. @@ -181,6 +182,13 @@ def pack_diffusion_output_shm( if isinstance(output, DiffusionOutput): return _pack_diffusion_fields(output, d2h_stream=d2h_stream) + # DP multi-concurrency: {"dp_rank": int, "output": DiffusionOutput} + if isinstance(output, dict) and "dp_rank" in output and "output" in output: + inner = output["output"] + if isinstance(inner, DiffusionOutput): + output["output"] = _pack_diffusion_fields(inner) + return output + if _is_rpc_result_envelope(output): result = output.get("result") if isinstance(result, DiffusionOutput): @@ -211,6 +219,13 @@ def unpack_diffusion_output_shm(output: object) -> object: if isinstance(output, DiffusionOutput): return _unpack_diffusion_fields(output) + # DP multi-concurrency: {"dp_rank": int, "output": DiffusionOutput} + if isinstance(output, dict) and "dp_rank" in output and "output" in output: + inner = output["output"] + if isinstance(inner, DiffusionOutput): + output["output"] = _unpack_diffusion_fields(inner) + return output + if _is_rpc_result_envelope(output): result = output.get("result") if isinstance(result, DiffusionOutput): diff --git a/vllm_omni/diffusion/model_loader/diffusers_loader.py b/vllm_omni/diffusion/model_loader/diffusers_loader.py index d7770c266..df7aa2523 100644 --- a/vllm_omni/diffusion/model_loader/diffusers_loader.py +++ b/vllm_omni/diffusion/model_loader/diffusers_loader.py @@ -362,17 +362,42 @@ class DiffusersPipelineLoader: ) else: model = self._init_from_load_format(load_format, target_device, custom_pipeline_name, is_hsdp=False) - logger.debug("Loading weights on %s ...", load_device) - if load_format == "diffusers": - # DiffusersAdapterPipeline.load_weights() calls - # DiffusionPipeline.from_pretrained() internally; it does - # not use our native customized pipeline classes. - cast(DiffusersAdapterPipeline, model).load_weights() + + # Skip load_weights only for DLO+AllGather when the model + # supports mmap loading and no online quantization is active. + # This condition MUST match the gate in + # DistributedLayerwiseOffloadBackend.enable() so that the + # loader skips ⟺ enable() uses mmap. + from vllm_omni.diffusion.offloader.offload_plan import supports_mmap_loading + + _dist_offload = getattr(self.od_config, "enable_distributed_layerwise_offload", False) + _use_ag = getattr(self.od_config, "dlo_use_allgather", True) + _has_online_quant = self._has_online_quant(model) + _supports_mmap = supports_mmap_loading(model) + + _skip_load = _dist_offload and _use_ag and _supports_mmap and not _has_online_quant + + if _skip_load: + logger.info("DLO+AllGather active: skipping load_weights (will load via mmap in enable())") else: - self.load_weights(model) - # HSDP processes quantized weights before wrapping parameters as - # DTensors. The non-HSDP path can process them here as usual. - self._process_weights_after_loading(model, target_device) + if _dist_offload and _use_ag and _has_online_quant: + raise ValueError( + "Online quantization is incompatible with DLO+AllGather: " + "the sharding + AllGather mechanism flattens weights by " + "dtype, which breaks quantized weight/scale layouts. " + "Please use --dlo-no-use-allgather or disable online " + "quantization." + ) + if _dist_offload and _use_ag and not _supports_mmap: + logger.info( + "DLO+AllGather active but model does not support mmap: loading weights via regular loader." + ) + logger.debug("Loading weights on %s ...", load_device) + if load_format == "diffusers": + cast(DiffusersAdapterPipeline, model).load_weights() + else: + self.load_weights(model) + self._process_weights_after_loading(model, target_device) if offload_after_quant: model.to("cpu") diff --git a/vllm_omni/diffusion/offloader/__init__.py b/vllm_omni/diffusion/offloader/__init__.py index 7987da887..807fad7ed 100644 --- a/vllm_omni/diffusion/offloader/__init__.py +++ b/vllm_omni/diffusion/offloader/__init__.py @@ -8,24 +8,54 @@ from vllm_omni.diffusion.data import OmniDiffusionConfig from vllm_omni.platforms import current_omni_platform from .base import OffloadBackend, OffloadConfig, OffloadStrategy +from .block_discovery import get_blocks_attr_names, get_blocks_from_dit, set_blocks_attr_names +from .distributed_layerwise_backend import ( + DistributedLayerwiseOffloadBackend, + DistributedLayerwiseOffloadHook, + apply_distributed_block_hook, + remove_distributed_block_hook, +) from .layerwise_backend import LayerWiseOffloadBackend +from .offload_plan import OffloadPlan, get_offload_plan, supports_mmap_loading from .sequential_backend import ( ModelLevelOffloadBackend, apply_sequential_offload, remove_sequential_offload, ) +from .tensor_utils import ( + dtype_size, + is_dtensor, + is_materialized_tensor, + make_offload_placeholder, + set_tensor_storage, +) logger = init_logger(__name__) __all__ = [ "OffloadBackend", "OffloadConfig", + "OffloadPlan", "OffloadStrategy", "LayerWiseOffloadBackend", + "DistributedLayerwiseOffloadBackend", + "DistributedLayerwiseOffloadHook", "ModelLevelOffloadBackend", "apply_sequential_offload", "remove_sequential_offload", + "apply_distributed_block_hook", + "remove_distributed_block_hook", "get_offload_backend", + "get_offload_plan", + "supports_mmap_loading", + "get_blocks_attr_names", + "get_blocks_from_dit", + "set_blocks_attr_names", + "dtype_size", + "is_dtensor", + "is_materialized_tensor", + "make_offload_placeholder", + "set_tensor_storage", ] @@ -75,6 +105,8 @@ def get_offload_backend( return ModelLevelOffloadBackend(config, device) elif config.strategy == OffloadStrategy.LAYER_WISE: return LayerWiseOffloadBackend(config, device) + elif config.strategy == OffloadStrategy.DISTRIBUTED_LAYER_WISE: + return DistributedLayerwiseOffloadBackend(config, device) else: logger.error("Unknown offload strategy: %s", config.strategy) return None diff --git a/vllm_omni/diffusion/offloader/base.py b/vllm_omni/diffusion/offloader/base.py index e10438765..20b2f5e14 100644 --- a/vllm_omni/diffusion/offloader/base.py +++ b/vllm_omni/diffusion/offloader/base.py @@ -18,6 +18,7 @@ class OffloadStrategy(Enum): NONE = "none" MODEL_LEVEL = "model_level" # Sequential offloading between DiT and encoders LAYER_WISE = "layer_wise" # Block-level + DISTRIBUTED_LAYER_WISE = "distributed_layer_wise" # Block-level with DP sharding + H2D/AllGather overlap @dataclass @@ -25,13 +26,21 @@ class OffloadConfig: strategy: OffloadStrategy pin_cpu_memory: bool = True use_hsdp: bool = False + dp_size: int = 1 # derived from parallel_config, not user-configurable + dlo_use_allgather: bool = True # if True: shard + AllGather; if False: full weights H2D only + model_path: str | None = None # checkpoint path for mmap weight loading @classmethod def from_od_config(cls, od_config: OmniDiffusionConfig) -> "OffloadConfig": """Extract and validate offload settings from OmniDiffusionConfig. - For now, enforces mutual exclusion between model-level and layer-wise offloading. - Layer-wise takes priority if both are enabled. + Enforces mutual exclusion among the three offload strategies. + Distributed layer-wise takes the highest priority, then layer-wise, + then model-level. + + The ``dp_size`` is automatically derived from ``parallel_config`` — + it is NOT a user-configurable parameter. The distributed layerwise + offload works with whatever DP/SP parallelism is already set up. Args: od_config: OmniDiffusionConfig with offload settings @@ -41,13 +50,39 @@ class OffloadConfig: """ enable_cpu_offload = getattr(od_config, "enable_cpu_offload", False) enable_layerwise_offload = getattr(od_config, "enable_layerwise_offload", False) + enable_distributed_layerwise_offload = getattr(od_config, "enable_distributed_layerwise_offload", False) pin_cpu_memory = getattr(od_config, "pin_cpu_memory", True) parallel_config = getattr(od_config, "parallel_config", None) use_hsdp = getattr(parallel_config, "use_hsdp", False) if parallel_config else False - # Determine strategy (mutual exclusion, layer-wise takes priority) - if enable_layerwise_offload: + # Derive dp_size from parallel_config — not user-configurable. + # The offload adapts to whatever DP/SP is already configured. + dp_size = 1 + if parallel_config is not None: + dp_size = getattr(parallel_config, "data_parallel_size", 1) + # HSDP's fully_shard_degree also contributes to effective DP + hsdp_shard_size = getattr(parallel_config, "hsdp_shard_size", -1) if use_hsdp else -1 + hsdp_replicate_size = getattr(parallel_config, "hsdp_replicate_size", 1) if use_hsdp else 1 + if use_hsdp and hsdp_shard_size > 0: + dp_size = hsdp_shard_size * hsdp_replicate_size + + # When there is no DP but SP > 1, shard weights across SP ranks. + # AllGather reconstructs full weights per layer; each rank then + # computes on its SP portion of the sequence. This gives N× + # compute parallelism with 1/N H2D transfer, reusing the exact + # same AllGather code path — only the process group changes. + if dp_size <= 1: + sp_size = getattr(parallel_config, "sequence_parallel_size", 1) + if sp_size and sp_size > 1: + dp_size = sp_size + + # Determine strategy (mutual exclusion, distributed layer-wise takes priority) + if enable_distributed_layerwise_offload: + strategy = OffloadStrategy.DISTRIBUTED_LAYER_WISE + if enable_layerwise_offload or enable_cpu_offload: + logger.info("Distributed layer-wise offloading takes priority, disabling other offloading strategies.") + elif enable_layerwise_offload: strategy = OffloadStrategy.LAYER_WISE if enable_cpu_offload: logger.info( @@ -59,10 +94,37 @@ class OffloadConfig: else: strategy = OffloadStrategy.NONE + # When dlo_use_allgather=False, each rank loads full weights (no sharding, + # no AllGather). This avoids AllGather synchronization requirements + # (concurrent requests, dummy run skip) at the cost of N× CPU memory. + dlo_use_allgather = getattr(od_config, "dlo_use_allgather", True) + + # If dlo_use_allgather=False, force dp_size=1 (each rank independent) + if enable_distributed_layerwise_offload and not dlo_use_allgather: + dp_size = 1 + logger.info( + "Distributed layerwise offload: dlo_use_allgather=False, " + "each rank loads full weights (no shard, no AllGather)" + ) + + # HSDP already shards parameters into DTensors. Running distributed + # layerwise offload on top would shard each to_local() again, producing + # incorrect reconstruction after AllGather. Reject this combination. + if enable_distributed_layerwise_offload and use_hsdp and dlo_use_allgather: + raise ValueError( + "Distributed layerwise offload with AllGather is incompatible with " + "HSDP: HSDP parameters are already sharded DTensors, and the offloader " + "would double-shard them. Use --dlo-no-use-allgather (full weights per " + "rank) or disable HSDP." + ) + return cls( strategy=strategy, pin_cpu_memory=pin_cpu_memory, use_hsdp=use_hsdp, + dp_size=dp_size, + dlo_use_allgather=dlo_use_allgather, + model_path=getattr(od_config, "model", None), ) diff --git a/vllm_omni/diffusion/offloader/block_discovery.py b/vllm_omni/diffusion/offloader/block_discovery.py new file mode 100644 index 000000000..643485ed6 --- /dev/null +++ b/vllm_omni/diffusion/offloader/block_discovery.py @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Block discovery for layerwise offload. + +Shared between LayerWiseOffloadBackend and DistributedLayerwiseOffloadBackend. +""" + +from __future__ import annotations + +from torch import nn +from vllm.logger import init_logger + +logger = init_logger(__name__) + + +def get_blocks_attr_names(model: nn.Module) -> list[str]: + """Get block attribute names from model class.""" + attrs: list[str] = getattr(model.__class__, "_layerwise_offload_blocks_attrs", []) + + if not attrs: + old_attr = getattr(model.__class__, "_layerwise_offload_blocks_attr", None) + if old_attr is not None: + logger.warning( + "'_layerwise_offload_blocks_attr' is deprecated, " + "please use '_layerwise_offload_blocks_attrs' instead. " + "Example: _layerwise_offload_blocks_attrs = ['blocks']" + ) + attrs = [old_attr] if isinstance(old_attr, str) else list(old_attr) + + return attrs + + +def set_blocks_attr_names(model: nn.Module, names: list[str]) -> None: + if not hasattr(model.__class__, "_layerwise_offload_blocks_attrs"): + setattr(model.__class__, "_layerwise_offload_blocks_attrs", names) + + +def get_blocks_from_dit(model: nn.Module) -> tuple[list[str], list[nn.Module]]: + """Retrieve blocks and attribute names from provided DiT model.""" + blocks_attr_names = get_blocks_attr_names(model) + if not blocks_attr_names: + logger.warning( + f"No _layerwise_offload_blocks_attrs defined for {model.__class__.__name__}, " + "skipping distributed layerwise offloading" + ) + return [], [] + + blocks: list[nn.Module] = [] + for name in blocks_attr_names: + attr = getattr(model, name, None) + if attr is None: + raise AttributeError( + f"Attribute '{name}' declared in _layerwise_offload_blocks_attrs " + f"does not exist on model {model.__class__.__name__}" + ) + try: + attr_iter = iter(attr) + except TypeError: + if isinstance(attr, nn.Module): + logger.warning( + "Attribute '%s' on %s is not iterable; treating it as one block.", + name, + model.__class__.__name__, + ) + blocks.append(attr) + continue + + logger.warning( + "Attribute '%s' on %s is not iterable (got %s); skipping it.", + name, + model.__class__.__name__, + type(attr).__name__, + ) + else: + blocks.extend(attr_iter) + + if not blocks: + logger.warning( + "No blocks found in %s for %s, skipping distributed layerwise offloading", + blocks_attr_names, + model.__class__.__name__, + ) + return [], [] + + return blocks_attr_names, blocks diff --git a/vllm_omni/diffusion/offloader/distributed_layerwise_backend.py b/vllm_omni/diffusion/offloader/distributed_layerwise_backend.py new file mode 100644 index 000000000..797209781 --- /dev/null +++ b/vllm_omni/diffusion/offloader/distributed_layerwise_backend.py @@ -0,0 +1,1491 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Distributed Layerwise Offload backend with H2D + AllGather overlap. + +This module implements the RFC-1 "Distributed Layerwise Offload" mechanism that: + +* Shards model weights across DP ranks and stores only the local shard + (1/DP_size of full model) in host pinned memory per rank. +* Uses a fixed double-buffer scheme that keeps only two layers' worth of + weights on each device at any time. +* Asynchronously pipelines both H2D transfers and AllGather communications + on dedicated streams, fully overlapping them with computation. +* Is hardware-agnostic, supporting both NVIDIA GPU (CUDA) and Ascend NPU + (CANN) platforms via vLLM-Omni's platform abstraction layer. +""" + +from __future__ import annotations + +import os +from itertools import chain +from typing import Any + +import torch +import torch.distributed +from torch import nn +from vllm.logger import init_logger + +from vllm_omni.diffusion.hooks import HookRegistry, ModelHook +from vllm_omni.platforms import current_omni_platform + +from .base import OffloadBackend, OffloadConfig +from .block_discovery import get_blocks_from_dit +from .module_collector import ModuleDiscovery +from .offload_plan import OffloadPlan, get_offload_plan, supports_mmap_loading +from .tensor_utils import ( + dtype_size as _dtype_size, +) +from .tensor_utils import ( + is_materialized_tensor, + make_offload_placeholder, + set_tensor_storage, +) + +logger = init_logger(__name__) + +# Threshold (in MB) for deciding whether a non-block DiT submodule should +# use layerwise offload (streaming hooks) or be moved to GPU as a resident +# module. Submodules larger than this are offloaded to save HBM; smaller +# ones stay resident for lower latency. +_ON_DEMAND_THRESHOLD_MB = 1024 + + +class DistributedLayerwiseOffloadHook(ModelHook): + """Hook for distributed layerwise offloading with fixed double-buffer. + + Each rank stores only a shard of each block's weights on host memory. + Two device slots alternate: one holds current weights, one holds next weights. + H2D and AllGather run asynchronously on dedicated streams, overlapped + with computation. + + Supports both NVIDIA GPU (CUDA) and Ascend NPU (CANN) platforms. + """ + + _HOOK_NAME = "distributed_layerwise_offload" + + def __init__( + self, + next_block: nn.Module, + device: torch.device, + dp_group: torch.distributed.ProcessGroup | None, + dp_size: int, + rank: int, + copy_stream: Any | None = None, + comm_stream: Any | None = None, + pin_memory: bool = True, + shared_buffers: list[dict[torch.dtype, torch.Tensor] | None] | None = None, + ): + assert isinstance(next_block, nn.Module), "transformer block must be type `torch.nn.Module`" + + self.next_block = next_block + self.device = device + self.dp_group = dp_group + self.dp_size = dp_size + self.rank = rank + self.pin_memory = pin_memory + + self.copy_stream = copy_stream or current_omni_platform.Stream() + self.comm_stream = comm_stream or current_omni_platform.Stream() + + # Double buffers: either shared (from backend) or self-allocated (lazy) + if shared_buffers is not None: + self.gpu_buffers: list[dict[torch.dtype, torch.Tensor] | None] = shared_buffers + self._owns_buffers = False + else: + self.gpu_buffers = [None, None] + self._owns_buffers = True + self.ready_events: list[Any | None] = [None, None] + + # Sharded host weights for the next block, keyed by dtype + self.cpu_shards: dict[torch.dtype, torch.Tensor] = {} + self.metadata: dict[torch.dtype, list[dict[str, Any]]] = {} + + # Current slot index (0 or 1). Updated dynamically by the previous + # hook's prefetch_layer call via _prefetched_slot. This ensures + # correct slot tracking for ALL block counts (including odd N). + self.current_slot = 0 + self._prefetched_slot: int | None = None + + # Backward link to previous hook for fallback (cache-dit skip) + self._prev_hook: DistributedLayerwiseOffloadHook | None = None + + # Marks the first hook in a shared-buffer group. When multiple DiT + # groups share the same 2 GPU buffers, another group may have + # overwritten this group's slot between forwards. The first block + # must sync-prefetch on entry to ensure it loads the correct + # weights, even if is_materialized sees a non-empty tensor left by + # the other group. + self._is_group_first: bool = False + + # Group ID for per-slot contamination tracking. Shared across all + # hooks in the same group. When a hook prefetches into a slot, it + # stamps the slot with its group ID. On group entry, the first + # hook checks whether the slot was last written by its own group + # (tail hook's async prefetch) — if so, skip the sync-prefetch. + self._group_id: int = -1 + self._shared_slot_group: list[int] | None = None # [-1, -1], shared + + # Parameters/buffers of the current and next blocks + self.block_parameters: dict[str, nn.Parameter] = {} + self.block_buffers: dict[str, torch.Tensor] = {} + self.next_block_parameters: dict[str, nn.Parameter] = {} + self.next_block_buffers: dict[str, torch.Tensor] = {} + + # Per-block synchronization primitive: set after H2D copy completes. + self._prefetch_done: Any | None = None + + # Shared shard (AllGather input) buffers — assigned by backend. + self.gpu_shard_buffers: list[dict[torch.dtype, torch.Tensor] | None] = [None, None] + + # Pending async AllGather work (prevent GC before completion). + self._pending_work: Any | None = None + self._cached_repoint: list | None = None + + # ------------------------------------------------------------------ # + # DTensor helpers (shared with LayerwiseOffloadHook) # + # ------------------------------------------------------------------ # + + def initialize_hook(self, module: nn.Module) -> nn.Module: + module = super().initialize_hook(module) + + self.block_parameters = dict(module.named_parameters()) + self.block_buffers = dict(module.named_buffers()) + + self.next_block_parameters = dict(self.next_block.named_parameters()) + self.next_block_buffers = dict(self.next_block.named_buffers()) + + # Shard next block's weights and store local shard in pinned CPU memory + self.cpu_shards, self.metadata = self._shard_and_pin( + self.next_block_parameters, + self.next_block_buffers, + self.dp_size, + self.rank, + self.pin_memory, + ) + + # Allocate device buffers only if not using shared buffers from backend + if self._owns_buffers: + self._allocate_device_buffers() + + # Cache parameter re-pointing metadata to avoid per-layer dict lookups. + self._cached_repoint = [] + for slot in range(2): + repoint = [] + for dtype, metas in self.metadata.items(): + for m in metas: + target = ( + self.next_block_parameters[m["name"]] + if m["name"] in self.next_block_parameters + else self.next_block_buffers[m["name"]] + ) + repoint.append((target, dtype, m["offset"], m["numel"], m["shape"])) + self._cached_repoint.append(repoint) + + # Pre-compute AG output sizes (avoid sum() per layer). + self._ag_output_sizes: dict[torch.dtype, int] = {} + for dtype, metas in self.metadata.items(): + total_numel = sum(m["numel"] for m in metas) + shard_numel = self.cpu_shards[dtype].numel() + self._ag_output_sizes[dtype] = shard_numel * self.dp_size if self.dp_size > 1 else total_numel + + return module + + @staticmethod + def _shard_and_pin( + params: dict[str, nn.Parameter], + bufs: dict[str, torch.Tensor], + dp_size: int, + rank: int, + pin_memory: bool, + ) -> tuple[dict[torch.dtype, torch.Tensor], dict[torch.dtype, list[dict[str, Any]]]]: + """Flatten params+buffers by dtype, split into DP shards, store local shard. + + Each rank stores only ``1/dp_size`` of the total weights. The full + tensor is reconstructed at runtime via AllGather. + """ + dtype_grouped: dict[torch.dtype, dict[str, torch.Tensor]] = {} + dtype_metadata: dict[torch.dtype, list[dict[str, Any]]] = {} + + for name, param_or_buf in chain(params.items(), bufs.items()): + dtype = param_or_buf.dtype + if dtype not in dtype_grouped: + dtype_grouped[dtype] = {} + dtype_grouped[dtype][name] = param_or_buf + + cpu_shards: dict[torch.dtype, torch.Tensor] = {} + + for dtype, name2weights in dtype_grouped.items(): + # Resolve local tensors (handle DTensor via to_local) + weights_with_local = [] + for name, t in name2weights.items(): + local_t = t.to_local() if hasattr(t, "to_local") else t + weights_with_local.append((name, t, local_t)) + + total_numel = sum(local.numel() for _, _, local in weights_with_local) + + # Equal-sized shards (ceil division) for all_gather_into_tensor + shard_size = (total_numel + dp_size - 1) // dp_size # ceil + shard_start = rank * shard_size + shard_end = min(shard_start + shard_size, total_numel) + + # Allocate ONLY the shard (1/dp_size), zero-padded to ceil. + # Avoids materialising the full block on CPU. + shard = torch.zeros(shard_size, dtype=dtype, device="cpu") + + current_offset = 0 + for name, original_tensor, local_tensor in weights_with_local: + numel = local_tensor.numel() + if dtype not in dtype_metadata: + dtype_metadata[dtype] = [] + # Offsets remain relative to the FULL flattened buffer + # (needed for correct AllGather reconstruction). + dtype_metadata[dtype].append( + { + "name": name, + "offset": current_offset, + "numel": numel, + "shape": local_tensor.shape, + } + ) + + # Copy ONLY the portion within [shard_start, shard_end) + overlap_start = max(current_offset, shard_start) + overlap_end = min(current_offset + numel, shard_end) + if overlap_start < overlap_end: + src_start = overlap_start - current_offset + src_end = overlap_end - current_offset + dst_start = overlap_start - shard_start + dst_end = overlap_end - shard_start + shard[dst_start:dst_end].copy_(local_tensor.flatten()[src_start:src_end]) + + # Replace original tensor with placeholder (frees CPU storage) + set_tensor_storage( + original_tensor, + make_offload_placeholder(original_tensor), + ) + current_offset += numel + + if pin_memory: + shard = shard.pin_memory() + + cpu_shards[dtype] = shard + + return cpu_shards, dtype_metadata + + def _allocate_device_buffers(self) -> None: + """Pre-allocate exactly two device buffers (one per slot).""" + for slot in range(2): + gpu_weights: dict[torch.dtype, torch.Tensor] = {} + for dtype, metas in self.metadata.items(): + total_numel = sum(m["numel"] for m in metas) + # AllGather output = dp_size * shard_size (padded) + padded = total_numel + if self.dp_size > 1: + shard_sz = (total_numel + self.dp_size - 1) // self.dp_size + padded = shard_sz * self.dp_size + gpu_weights[dtype] = torch.empty(padded, dtype=dtype, device=self.device) + self.gpu_buffers[slot] = gpu_weights + + @property + def is_materialized(self) -> bool: + """Check whether this block's parameters hold real data on device.""" + for param in self.block_parameters.values(): + return is_materialized_tensor(param) + return True + + # ------------------------------------------------------------------ # + # Prefetch: H2D + AllGather (overlapped on dedicated streams) # + # ------------------------------------------------------------------ # + + @torch.compiler.disable + def prefetch_layer(self, slot: int, non_blocking: bool = True) -> None: + """Prepare next block's weights into the shared device buffer for *slot*. + + Uses the pre-allocated ``self.gpu_buffers[slot]`` instead of + allocating fresh tensors every layer. This enforces the fixed + double-buffer memory bound (exactly 2 blocks on device). + """ + self.copy_stream.wait_stream(current_omni_platform.current_stream()) + + evt = current_omni_platform.Event() + gpu_weights = self.gpu_buffers[slot] + assert gpu_weights is not None, f"gpu_buffers[{slot}] not allocated" + + if self.dp_size <= 1 or self.dp_group is None: + with current_omni_platform.stream(self.copy_stream): + for dtype, cpu_shard in self.cpu_shards.items(): + gw = gpu_weights[dtype] + gw[: cpu_shard.numel()].copy_(cpu_shard, non_blocking=non_blocking) + evt.record(self.copy_stream) + else: + gpu_shards: dict[torch.dtype, torch.Tensor] = {} + shard_bufs = self.gpu_shard_buffers[slot] + assert shard_bufs is not None, f"gpu_shard_buffers[{slot}] not allocated" + with current_omni_platform.stream(self.copy_stream): + for dtype, cpu_shard in self.cpu_shards.items(): + gpu_shard = shard_bufs[dtype][: cpu_shard.numel()] + gpu_shard.copy_(cpu_shard, non_blocking=non_blocking) + gpu_shards[dtype] = gpu_shard + + self.comm_stream.wait_stream(self.copy_stream) + with current_omni_platform.stream(self.comm_stream): + for dtype, local_shard in gpu_shards.items(): + gw = gpu_weights[dtype] + torch.distributed.all_gather_into_tensor( + gw, + local_shard, + group=self.dp_group, + ) + evt.record(self.comm_stream) + + self.ready_events[slot] = evt + self._prefetch_done = evt + self._prefetched_slot = slot + + # Stamp the slot with this hook's group ID so that group-first + # hooks can detect whether another group has overwritten the slot. + if self._shared_slot_group is not None: + self._shared_slot_group[slot] = self._group_id + + # Re-point using cached metadata (avoids per-layer dict lookups). + for target, dtype, offset, numel, shape in self._cached_repoint[slot]: + set_tensor_storage( + target, + gpu_weights[dtype][offset : offset + numel].view(shape), + ) + + def get_weights(self, slot: int) -> dict[torch.dtype, torch.Tensor] | None: + """Wait for AllGather completion and return full weights for the slot. + + The ready event for this slot was set by the *previous* hook's + prefetch_layer (which prefetched THIS block's weights into the + shared buffer). This hook's own ready_events[slot] is None because + it never prefetched into this slot itself. Fall back to the + previous hook's event to ensure the compute stream waits for the + AllGather (or H2D) to complete before reading the buffer. + """ + evt = self.ready_events[slot] + if evt is None and self._prev_hook is not None: + evt = self._prev_hook.ready_events[slot] + if evt is not None: + current_omni_platform.current_stream().wait_event(evt) + return self.gpu_buffers[slot] + + # ------------------------------------------------------------------ # + # Offload: free device memory for current block # + # ------------------------------------------------------------------ # + + @torch.compiler.disable + def offload_layer(self) -> None: + """Free GPU memory for current block by replacing tensors with placeholders.""" + evt = self._prefetch_done + if evt is not None: + current_omni_platform.current_stream().wait_event(evt) + self._prefetch_done = None + + for _, param in self.block_parameters.items(): + set_tensor_storage(param, make_offload_placeholder(param)) + for _, buf in self.block_buffers.items(): + set_tensor_storage(buf, make_offload_placeholder(buf)) + + # ------------------------------------------------------------------ # + # ModelHook interface # + # ------------------------------------------------------------------ # + + def pre_forward(self, module: nn.Module, *args: Any, **kwargs: Any) -> tuple[tuple, dict]: + # Dynamic slot tracking: read current_slot from the previous hook's + # _prefetched_slot (the slot where this block's data was actually + # loaded). This corrects the initial i%2 assignment for odd block + # counts, where the circular tail→head prefetch would otherwise + # collide with the head's read slot. + if self._prev_hook is not None and self._prev_hook._prefetched_slot is not None: + self.current_slot = self._prev_hook._prefetched_slot + + # Group-first hook: check whether the shared buffer slot was + # overwritten by another group since our last forward. If the + # slot still contains our own data (from the tail hook's async + # prefetch), skip the sync-prefetch and just wait for the event. + if self._is_group_first and self._prev_hook is not None: + slot_contaminated = True + if self._shared_slot_group is not None: + slot_contaminated = self._shared_slot_group[self.current_slot] != self._group_id + if slot_contaminated: + # Another group (or no group) wrote to our slot — re-fetch + self._prev_hook.prefetch_layer(self.current_slot, non_blocking=False) + # Always wait for data to be ready (handles both sync and async paths) + self._prev_hook.get_weights(self.current_slot) + elif not self.is_materialized and self._prev_hook is not None: + # Previous hook was skipped (e.g. by cache-dit), sync-prefetch + self._prev_hook.prefetch_layer(self.current_slot, non_blocking=False) + self._prev_hook.get_weights(self.current_slot) + + # Prefetch next layer into the other slot (overlapped with compute). + # No explicit get_weights() here — offload_layer() in the previous + # hook's post_forward already enqueued wait_event for the current + # layer's H2D/AllGather, which is sufficient to ensure compute_stream + # waits for weights before reading them. The redundant wait_event + # caused cascading NPU stream synchronization stalls (~10ms/layer). + next_slot = 1 - self.current_slot + self.prefetch_layer(next_slot, non_blocking=True) + + return args, kwargs + + def post_forward(self, module: nn.Module, output: Any) -> Any: + self.offload_layer() + return output + + +# ---------------------------------------------------------------------- # +# Module-level helpers # +# ---------------------------------------------------------------------- # + + +def apply_distributed_block_hook( + module: nn.Module, + next_block: nn.Module, + device: torch.device, + dp_group: torch.distributed.ProcessGroup | None, + dp_size: int, + rank: int, + copy_stream: Any | None = None, + comm_stream: Any | None = None, + pin_memory: bool = True, + shared_buffers: list[dict[torch.dtype, torch.Tensor] | None] | None = None, +) -> DistributedLayerwiseOffloadHook: + """Register a DistributedLayerwiseOffloadHook on *module*.""" + registry = HookRegistry.get_or_create(module) + hook = DistributedLayerwiseOffloadHook( + next_block=next_block, + device=device, + dp_group=dp_group, + dp_size=dp_size, + rank=rank, + copy_stream=copy_stream, + comm_stream=comm_stream, + pin_memory=pin_memory, + shared_buffers=shared_buffers, + ) + registry.register_hook(DistributedLayerwiseOffloadHook._HOOK_NAME, hook) + return hook + + +def remove_distributed_block_hook(module: nn.Module) -> None: + """Remove the distributed layerwise offload hook from *module*.""" + registry: HookRegistry | None = getattr(module, "_hook_registry", None) + if registry is not None: + registry.remove_hook(DistributedLayerwiseOffloadHook._HOOK_NAME) + logger.debug("Removed distributed offload hook from %s", module.__class__.__name__) + + +# ---------------------------------------------------------------------- # +# Backend # +# ---------------------------------------------------------------------- # + + +class DistributedLayerwiseOffloadBackend(OffloadBackend): + """Distributed layer-wise (block-level) offloading backend. + + Supports both GPU (CUDA) and NPU (CANN) platforms. + Device type is determined by the device passed to enable(). + + Each rank stores only a shard of each block's weights on host memory. + Two device slots alternate: one holds current weights, one holds next + weights. H2D and AllGather run asynchronously on dedicated streams, + overlapped with computation. + """ + + def __init__(self, config: OffloadConfig, device: torch.device): + super().__init__(config, device) + + self.copy_stream = current_omni_platform.Stream() + self.comm_stream = current_omni_platform.Stream() + self.dp_group: torch.distributed.ProcessGroup | None = None + self.dp_size = config.dp_size + self.rank = 0 + self._blocks: list[list[nn.Module]] = [] + self._all_hook_groups: list[list[DistributedLayerwiseOffloadHook]] = [] + + def _load_weights_via_mmap(self, pipeline: nn.Module, modules) -> None: + """Load DiT weights from safetensors via mmap views (no RSS). + + When the transformer is created on meta device, this method + replaces meta params with mmap views of the checkpoint files. + The views point to OS page cache (shared across ranks), so + no private copy is created. _shard_and_pin then copies only + the 1/dp_size shard from the mmap view to a private buffer. + + Non-DiT modules (VAE, encoders) are NOT affected — they were + created on CPU with real weights via from_pretrained. + """ + import glob + import json + + from safetensors import safe_open + + model_path = self.config.model_path + if not model_path: + logger.warning("No model_path for mmap weight loading, skipping") + return + + # Resolve HF repo ID to local snapshot path. + # If model_path is a local directory, use it directly; otherwise + # download the safetensors index + weight files via HuggingFace Hub. + if not os.path.isdir(model_path): + from vllm.model_executor.model_loader.weight_utils import ( + download_weights_from_hf, + ) + + logger.info("model_path %s is not local, downloading from HF", model_path) + model_path = download_weights_from_hf( + model_name_or_path=model_path, + cache_dir=None, + allow_patterns=["*.safetensors", "*.safetensors.index.json"], + ) + + # Build {checkpoint_key: file_path} from safetensors index + weight_map: dict[str, str] = {} + for idx_file in glob.glob(os.path.join(model_path, "**", "*.safetensors.index.json"), recursive=True): + idx_dir = os.path.dirname(idx_file) + with open(idx_file) as f: + idx = json.load(f) + for key, filename in idx.get("weight_map", {}).items(): + weight_map[key] = os.path.join(idx_dir, filename) + + if not weight_map: + single_files = glob.glob(os.path.join(model_path, "**", "model.safetensors"), recursive=True) + for sf in single_files: + from safetensors import safe_open as _safe_open + + f = _safe_open(sf, framework="pt", device="cpu") + for key in f.keys(): + weight_map[key] = sf + logger.info("Found single-file safetensors: %s (%d keys)", sf, len(f.keys())) + + if not weight_map: + raise RuntimeError( + "Distributed layerwise offload could not find safetensors " + f"weights at {model_path}. Expected *.safetensors.index.json " + "or model.safetensors. Ensure the checkpoint path is correct." + ) + + logger.info("Loading DiT weights via mmap (meta → page cache, no RSS): %d keys", len(weight_map)) + + # --- Convert DiT modules to meta device --- + # The transformer was created normally (with random weights and + # correct non-persistent buffers from __init__). We convert it + # to meta device to release the random weights (they're useless), + # then load real weights via mmap. + # + # Non-persistent buffers (e.g. RoPE inv_freq, timestep freqs) are + # NOT in the checkpoint — they are computed from formulas in + # __init__. Save them before meta conversion and restore after + # mmap loading, so we don't need model-specific buffer rebuild code. + # + # Fail closed for unsupported configurations BEFORE to_empty. + # The mmap path bypasses AutoWeightsLoader, which means TP-aware + # weight_loader callbacks (fused QKV, row-parallel, etc.) are not + # invoked. When TP > 1, these callbacks shard weights across TP + # ranks — skipping them produces incorrect weights. + # Check the actual TP world size from parallel_state (initialized + # before enable()). Note: params may have custom weight_loader + # attributes even at TP=1 (e.g. QKVParallelLinear), so we cannot + # reject based on attribute presence alone. + try: + from vllm.distributed.parallel_state import ( + get_tensor_model_parallel_world_size, + ) + + tp_world = get_tensor_model_parallel_world_size() + except Exception: + tp_world = 1 + if tp_world > 1: + raise ValueError( + "Distributed layerwise offload with mmap loading does not " + "support Tensor Parallel (TP > 1). TP-aware weight_loader " + "callbacks are bypassed by the mmap path. Use DP or SP " + "instead of TP, or disable distributed layerwise offload." + ) + + # + # Important: when DiT modules are nested (e.g. transformer contains + # transformer.language_model), to_empty("meta") on the parent + # converts the child's buffers too. So we must save ALL buffers + # from ALL DiT modules BEFORE any to_empty call. + saved_buffers: dict[int, dict[str, torch.Tensor]] = {} + for dit_module in modules.dits: + bufs: dict[str, torch.Tensor] = {} + for name, buf in dit_module.named_buffers(): + # Check if this buffer is non-persistent on its OWNING module + owner = dit_module + parts = name.split(".") + for part in parts[:-1]: + owner = getattr(owner, part) + buf_name = parts[-1] + is_non_persistent = buf_name in owner._non_persistent_buffers_set + # Save if non-persistent OR already meta (shouldn't happen + # before to_empty, but be safe) + if is_non_persistent: + bufs[name] = buf.detach().clone() + saved_buffers[id(dit_module)] = bufs + + # Now convert all DiT modules to meta (after saving all buffers). + # Skip modules that are already meta (happens when a parent DiT + # module contains a child DiT module — to_empty on the parent + # already converted the child). + for dit_module in modules.dits: + if any(p.is_meta for p in dit_module.parameters()): + logger.info( + "%s already on meta device (skipping to_empty, %d buffers saved)", + dit_module.__class__.__name__, + len(saved_buffers.get(id(dit_module), {})), + ) + continue + dit_module.to_empty(device="meta") + logger.info( + "Converted %s to meta device (%d non-persistent buffers saved)", + dit_module.__class__.__name__, + len(saved_buffers.get(id(dit_module), {})), + ) + + # Build reverse mapping {model_param_name: (ckpt_key, file_path)} using + # the pipeline's _remap_ckpt_key (if available). This handles the + # model-specific checkpoint key remapping (e.g., Cosmos3's + # layers.0.mlp_moe_gen.gate_proj → transformer.gen_layers.0.mlp.gate_proj). + remap_fn = getattr(type(pipeline), "_remap_ckpt_key", None) + model_to_ckpt: dict[str, tuple[str, str]] = {} + if remap_fn is not None: + for ckpt_key, file_path in weight_map.items(): + model_name = remap_fn(ckpt_key) + if model_name is not None: + model_to_ckpt[model_name] = (ckpt_key, file_path) + logger.info( + "Built reverse remap: %d model params from %d checkpoint keys", len(model_to_ckpt), len(weight_map) + ) + else: + # No remap function — use checkpoint keys directly + for ckpt_key, file_path in weight_map.items(): + model_to_ckpt[ckpt_key] = (ckpt_key, file_path) + + # Collect all meta params in DiT modules + dit_ids = set() + for dit_module in modules.dits: + dit_ids.update(id(m) for m in dit_module.modules()) + + # Cache open file handles + file_cache: dict[str, Any] = {} + loaded = 0 + skipped = 0 + loaded_names: set[str] = set() + + # Discover blocks first so we can distinguish block params (deferred + # to _shard_and_pin) from non-block params (loaded here as mmap views). + block_module_ids: set[int] = set() + for dit_module in modules.dits: + blocks_attr_names, blocks = get_blocks_from_dit(dit_module) + for block in blocks: + block_module_ids.update(id(m) for m in block.modules()) + + for name, param in pipeline.named_parameters(): + if not (hasattr(param, "is_meta") and param.is_meta): + continue + + # Check if this param is inside a block (will be handled by + # _shard_and_pin via mmap views in section 2 below). + parent = pipeline + parts = name.split(".") + is_block_param = False + for part in parts[:-1]: + try: + if part.isdigit(): + parent = parent[int(part)] + else: + parent = getattr(parent, part) + except (AttributeError, IndexError, TypeError): + break + if id(parent) in block_module_ids: + is_block_param = True + break + + if is_block_param: + # Block param — will be loaded as mmap view in section 2 + continue + + # Non-block param (merger, patch_embed, norm, proj_out, etc.) + # Look up in reverse remap mapping (handles Cosmos3 key remapping). + entry = model_to_ckpt.get(name) + if entry is None: + skipped += 1 + continue + + ckpt_key, file_path = entry + if file_path not in file_cache: + file_cache[file_path] = safe_open(file_path, framework="pt", device="cpu") + f = file_cache[file_path] + tensor = f.get_tensor(ckpt_key) + + # Replace meta param with mmap view (no copy!) + parent = pipeline + for part in parts[:-1]: + if part.isdigit(): + parent = parent[int(part)] + else: + parent = getattr(parent, part) + parent._parameters[parts[-1]] = torch.nn.Parameter(tensor, requires_grad=param.requires_grad) + loaded += 1 + loaded_names.add(name) + + logger.info("Mmap weight loading: %d non-block params loaded, %d skipped", loaded, skipped) + + # Now load DiT block params from safetensors using mmap views. + # Each block param is assigned an mmap view (no RSS). + # _shard_and_pin will later copy the shard portion to a private buffer. + block_loaded = 0 + for dit_idx, dit_module in enumerate(modules.dits): + dit_name = modules.dit_names[dit_idx] + blocks_attr_names, blocks = get_blocks_from_dit(dit_module) + if not blocks: + continue + + # Determine the blocks_attr for this DiT module (e.g. "layers" or "gen_layers") + blocks_attr = None + for attr_name in blocks_attr_names: + if hasattr(dit_module, attr_name): + candidate = getattr(dit_module, attr_name) + if isinstance(candidate, nn.ModuleList) and len(candidate) > 1: + blocks_attr = attr_name + break + + for block_idx, block in enumerate(blocks): + # Build the full model param prefix for this block + # e.g. "transformer.language_model.layers.0" or "transformer.gen_layers.0" + block_full_prefix = f"{dit_name}.{blocks_attr}.{block_idx}" + + for bname, bparam in block.named_parameters(): + if not (hasattr(bparam, "is_meta") and bparam.is_meta): + continue + + # Use reverse remap to find checkpoint key + full_param_name = f"{block_full_prefix}.{bname}" + entry = model_to_ckpt.get(full_param_name) + if entry is None: + skipped += 1 + continue + + ckpt_key, file_path = entry + if file_path not in file_cache: + file_cache[file_path] = safe_open(file_path, framework="pt", device="cpu") + f = file_cache[file_path] + tensor = f.get_tensor(ckpt_key) + + # Replace meta param with mmap view + parent = block + bparts = bname.split(".") + for part in bparts[:-1]: + if part.isdigit(): + parent = parent[int(part)] + else: + parent = getattr(parent, part) + parent._parameters[bparts[-1]] = torch.nn.Parameter(tensor, requires_grad=bparam.requires_grad) + block_loaded += 1 + loaded_names.add(full_param_name) + + logger.info("Mmap block loading: %d params loaded, %d skipped", block_loaded, skipped) + + # Strict validation: check that all meta params were loaded. + # This replaces the validation that AutoWeightsLoader.load_weights() + # performs in the regular load path. + remaining_meta = [ + name for name, param in pipeline.named_parameters() if hasattr(param, "is_meta") and param.is_meta + ] + if remaining_meta: + # Filter out params inside submodules that will be loaded later + # by _load_module_weights_from_mmap (non-block submodules) + dit_param_names = set() + for dit_module in modules.dits: + for pname, _ in dit_module.named_parameters(): + dit_param_names.add(pname) + truly_missing = [n for n in remaining_meta if n in dit_param_names] + if truly_missing: + logger.warning( + "Mmap loading: %d params still on meta device after " + "loading (first 5: %s). These may be loaded later by " + "_load_module_weights_from_mmap or are expected to be " + "meta (e.g. DTensor placeholders).", + len(truly_missing), + truly_missing[:5], + ) + + # Keep file handles open — _shard_and_pin will read from the mmap views. + # They will be released after _shard_and_pin completes (when params are + # replaced with offload placeholders). + self._mmap_file_cache = file_cache + self._mmap_model_to_ckpt = model_to_ckpt + + # The regular loader runs model-specific post-load transforms after + # assigning checkpoint tensors. The mmap path bypasses that loader, so + # preserve the same lifecycle for transforms such as Cosmos3's fp32 + # timestep embedder. + for dit_module in modules.dits: + post_load_weights = getattr(dit_module, "post_load_weights", None) + if callable(post_load_weights): + post_load_weights() + + # Call validate_loaded_weights if the model defines it. + # This preserves the sound-weight and action-weight validation + # that AutoWeightsLoader.load_weights() triggers in the regular + # load path (e.g. Cosmos3 checks for missing audio/action weights). + validate = getattr(dit_module, "validate_loaded_weights", None) + if callable(validate): + validate(loaded_names) + + # Restore non-persistent buffers that were saved before meta + # conversion. These buffers (e.g. RoPE inv_freq, timestep + # freqs) are computed from formulas in __init__ and are not + # present in the checkpoint. By saving and restoring them + # generically, we avoid model-specific buffer rebuild code. + bufs = saved_buffers.get(id(dit_module), {}) + for name, buf in bufs.items(): + parent = dit_module + parts = name.split(".") + for part in parts[:-1]: + if part.isdigit(): + parent = parent[int(part)] + else: + parent = getattr(parent, part) + # Place on the same device as the module's parameters + # Place on the offloader's target device (not the module's + # current device, which may still be meta at this point). + parent._buffers[parts[-1]] = buf.to(self.device) + + def _load_module_weights_from_mmap(self, module: nn.Module, dit_name: str, child_name: str) -> None: + """Load a non-block DiT submodule's weights from safetensors via mmap.""" + from safetensors import safe_open + + if not hasattr(self, "_mmap_file_cache") or not self._mmap_file_cache: + return + + file_cache = self._mmap_file_cache + model_to_ckpt = getattr(self, "_mmap_model_to_ckpt", {}) + loaded = 0 + + for pname, param in module.named_parameters(): + if not (hasattr(param, "is_meta") and param.is_meta): + continue + + # Build full model param name and look up in reverse remap + full_name = f"{dit_name}.{child_name}.{pname}" + entry = model_to_ckpt.get(full_name) + if entry is None: + # Try without dit_name prefix + full_name = f"{child_name}.{pname}" + entry = model_to_ckpt.get(full_name) + if entry is None: + continue + + ckpt_key, file_path = entry + if file_path not in file_cache: + file_cache[file_path] = safe_open(file_path, framework="pt", device="cpu") + f = file_cache[file_path] + tensor = f.get_tensor(ckpt_key) + + # Replace meta param with mmap view + parent = module + parts = pname.split(".") + for part in parts[:-1]: + if part.isdigit(): + parent = parent[int(part)] + else: + parent = getattr(parent, part) + parent._parameters[parts[-1]] = torch.nn.Parameter(tensor, requires_grad=param.requires_grad) + loaded += 1 + + if loaded > 0: + logger.info("Loaded %d params for submodule %s.%s via mmap", loaded, dit_name, child_name) + + def _init_dp_group(self) -> None: + """Reuse the process group initialized by parallel_state. + + When DP > 1, uses the DP group (handles strided groups with SP/CFG/PP). + When DP = 1 but SP > 1 (and dp_size was set to sp_size in OffloadConfig), + uses the SP group so weights are sharded across SP ranks. + """ + if self.dp_size <= 1: + logger.info("Distributed layerwise offload: dp_size=1, running without AllGather") + self.dp_group = None + return + + if not torch.distributed.is_initialized(): + raise RuntimeError( + "torch.distributed is not initialized. " + "Distributed layerwise offload with dp_size > 1 requires " + "an initialized process group." + ) + + from vllm_omni.diffusion.distributed.parallel_state import ( + get_data_parallel_world_size, + get_dp_group, + ) + + # Determine which parallel group to use for sharding. + # When data_parallel_size > 1, use the DP group. + # When data_parallel_size == 1 but dp_size > 1 (set from sp_size in + # OffloadConfig), use the SP group for weight sharding. + dp_world = get_data_parallel_world_size() + if dp_world > 1: + coord = get_dp_group() + else: + from vllm_omni.diffusion.distributed.parallel_state import get_sp_group + + coord = get_sp_group() + logger.info( + "Distributed layerwise offload: DP=1, using SP group (world_size=%d) for weight sharding", + coord.world_size, + ) + + self.dp_group = coord.device_group + self.rank = coord.rank_in_group + self.dp_size = coord.world_size + + logger.info( + "Distributed layerwise offload: dp_size=%d, rank_in_group=%d, global_rank=%d, group_ranks=%s", + self.dp_size, + self.rank, + coord.rank, + coord.ranks, + ) + + def _register_on_demand_hook(self, module: nn.Module, label: str) -> None: + """Move *module* to GPU permanently. + + Previously, on-demand hooks moved the module to GPU before forward + and back to CPU after forward (with synchronize + empty_cache). + However, _post_forward's global synchronize() and empty_cache() + disrupt the async prefetch pipeline (copy_stream / comm_stream) + used by distributed layerwise offload, causing garbled output. + + Moving the module to GPU permanently avoids this issue at the + cost of slightly higher HBM usage (VAE/encoders are small). + """ + module.to(self.device) + logger.info("Moved %s (%s) to GPU (resident)", label, module.__class__.__name__) + + def _try_layerwise_offload_submodule(self, module: nn.Module, name: str, plan: OffloadPlan | None = None) -> bool: + """Try to apply layerwise offload to a large submodule's blocks. + + Resolution order: + 1. OffloadPlan.offload_submodules (declarative, if plan is provided) + 2. Heuristic search for common block-list attributes + + Returns True if layerwise offload was applied, False otherwise. + """ + from operator import attrgetter + + blocks = None + blocks_attr = None + + # 1. Check OffloadPlan first (declarative — no guessing) + if plan is not None and name in plan.offload_submodules: + attr_name = plan.offload_submodules[name] + try: + candidate = attrgetter(attr_name)(module) + if isinstance(candidate, nn.ModuleList) and len(candidate) > 1: + blocks = candidate + blocks_attr = attr_name + except AttributeError: + logger.warning( + "OffloadPlan declared block attr '%s' for submodule '%s' " + "but attribute not found — falling back to heuristic", + attr_name, + name, + ) + + # 2. Fallback: heuristic search + if blocks is None: + for attr_name in ("layers", "blocks", "h", "model.layers"): + try: + candidate = attrgetter(attr_name)(module) + except AttributeError: + continue + if isinstance(candidate, nn.ModuleList) and len(candidate) > 1: + blocks = candidate + blocks_attr = attr_name + break + + if blocks is None: + return False + + num_blocks = len(blocks) + logger.info( + "Distributed layerwise offload for submodule '%s.%s' (%d blocks, %.0f MB total, dp_size=%d)", + name, + blocks_attr, + num_blocks, + sum(p.nelement() * p.element_size() for p in module.parameters()) / 1048576, + self.dp_size, + ) + + # Move non-block parts of the submodule to GPU (small: embeddings, norms) + for child_name, child in module.named_children(): + if child_name != blocks_attr: + child.to(self.device) + + # Apply distributed hooks (1/4 sharding + AllGather, same as DiT) + # Pass shared_buffers=[None,None] to defer per-hook allocation + # (prevents OOM on large models where N hooks × 2 buffers >> HBM) + last_block, first_block = blocks[-1], blocks[0] + last_hook = apply_distributed_block_hook( + last_block, + first_block, + self.device, + self.dp_group, + self.dp_size, + self.rank, + self.copy_stream, + self.comm_stream, + self.config.pin_cpu_memory, + shared_buffers=[None, None], + ) + sub_hooks = [last_hook] + for i, block in enumerate(blocks[:-1]): + next_block = blocks[(i + 1) % num_blocks] + hook = apply_distributed_block_hook( + block, + next_block, + self.device, + self.dp_group, + self.dp_size, + self.rank, + self.copy_stream, + self.comm_stream, + self.config.pin_cpu_memory, + shared_buffers=[None, None], + ) + sub_hooks.append(hook) + + # Wire backward references + slot alternation + for i in range(len(sub_hooks)): + sub_hooks[i]._prev_hook = sub_hooks[i - 1] + # Assign slots in list order: sub_hooks = [last_hook, block0, ..., blockN-2] + # This ensures last_hook.current_slot != block0_hook.current_slot, + # so the circular prefetch (last_hook -> block0) writes to a + # different slot than block0 reads from. Correct for ALL N. + for i, hook in enumerate(sub_hooks): + hook.current_slot = i % 2 + # Mark block0_hook (index 1) as group-first — it must sync-prefetch + # on entry because another group may have overwritten the shared slot. + if len(sub_hooks) > 1: + sub_hooks[1]._is_group_first = True + + # Defer buffer allocation and prefetch to enable() unified allocation + self._all_hook_groups.append(sub_hooks) + self._blocks.append(blocks) + return True + + def enable(self, pipeline: nn.Module) -> None: + if self.enabled: + logger.warning("DistributedLayerwiseOffloadBackend already enabled") + return + + self._on_demand_shard_infos: list[dict] = [] + self._on_demand_handles: list[Any] = [] + + # Initialize DP group (if not already done by early init) + if self.dp_group is None and self.dp_size > 1: + self._init_dp_group() + + modules = ModuleDiscovery.discover(pipeline) + if not modules.dits: + logger.warning("No DiT/transformer modules found, skipping distributed layer-wise offloading") + return + + # Retrieve optional declarative OffloadPlan from the pipeline. + # When present, replaces heuristic block discovery. + plan = get_offload_plan(pipeline) + + # Load weights via mmap for DLO+AllGather. + # Gate condition MUST match diffusers_loader.py: + # supports_mmap_loading(pipeline) and not _has_online_quant + # When the gate is False, the loader has already loaded weights via + # regular load_weights() + _process_weights_after_loading(). + if self.config.dlo_use_allgather and self.dp_size > 1: + _has_online_quant = any( + getattr(getattr(module, "quant_method", None), "uses_meta_device", False) + for module in pipeline.modules() + ) + if _has_online_quant: + raise ValueError( + "Online quantization is incompatible with DLO+AllGather: " + "the sharding + AllGather mechanism flattens weights by " + "dtype, which breaks quantized weight/scale layouts. " + "Please use --dlo-no-use-allgather or disable online " + "quantization." + ) + if supports_mmap_loading(pipeline): + self._load_weights_via_mmap(pipeline, modules) + else: + logger.info("Weights loaded via regular loader — skipping mmap (model does not support mmap)") + + # Keep VAE/encoders on CPU; move to GPU on-demand via hooks. + # This saves ~4.3 GB HBM per card (VAE 1.3 + encoder 1.1 + sound 1.9) + # during the DiT forward pass. They are only needed briefly for + # text-encoding (before DiT) and VAE-decode (after DiT). + for enc in modules.encoders: + self._register_on_demand_hook(enc, "encoder") + for vae in modules.vaes: + self._register_on_demand_hook(vae, "vae") + + # Move resident modules to GPU (small modules needed every forward) + for name, module in zip(modules.resident_names, modules.resident_modules): + try: + module.to(self.device) + except Exception as exc: + logger.debug("Failed to move resident module %s to GPU: %s", name, exc) + + logger.info("Applying distributed layer-wise offloading on %s", modules.dit_names) + + # Collect all DiT module objects to detect submodules that are + # already handled as a separate DiT module (avoids duplicate hooks). + all_dit_modules = set(id(m) for m in modules.dits) + + # Apply hooks for each DiT module + for i, dit_module in enumerate(modules.dits): + dit_name = modules.dit_names[i] + logger.info(f"Applying hooks on {dit_name} ({dit_module.__class__.__name__})") + + blocks_attr_names, blocks = get_blocks_from_dit(dit_module) + + if not blocks: + logger.warning( + "Target layers (blocks) not found. Skipping offloading on %s (%s)", + dit_name, + dit_module.__class__.__name__, + ) + dit_module.to(self.device) + continue + + num_blocks = len(blocks) + if num_blocks <= 1: + logger.warning( + "#Target layers (blocks) <= 1. Skipping offloading on %s (%s)", + dit_name, + dit_module.__class__.__name__, + ) + dit_module.to(self.device) + continue + + # Move non-block modules to GPU (they stay resident). + # Large modules (> 1 GB) use on-demand hooks instead — they are + # only needed briefly (e.g. language_model for text encoding) + # and keeping them resident wastes HBM during the DiT loop. + _ON_DEMAND_THRESHOLD = _ON_DEMAND_THRESHOLD_MB # alias for readability + for name, m in dit_module.named_children(): + if name not in blocks_attr_names: + _has_meta = any((p.is_meta if hasattr(p, "is_meta") else False) for p in m.parameters()) + if _has_meta: + # Meta params: load from safetensors before moving to GPU + self._load_module_weights_from_mmap(m, dit_name, name) + _mb = ( + sum( + (p.nelement() * p.element_size() if not (hasattr(p, "is_meta") and p.is_meta) else 0) + for p in m.parameters() + ) + / 1048576 + ) + if _mb > _ON_DEMAND_THRESHOLD: + # Skip if this submodule is already a separate DiT + # module (e.g. transformer.language_model when both + # "transformer.language_model" and "transformer" are + # in _dit_modules) — hooks already applied. + if id(m) in all_dit_modules: + logger.info("Submodule '%s' is already a DiT module, skipping layerwise offload", name) + elif self._try_layerwise_offload_submodule(m, name, plan): + pass # layerwise hooks applied + else: + self._register_on_demand_hook(m, name) + else: + try: + m.to(self.device) + except (NotImplementedError, RuntimeError): + # Meta params: load from safetensors via mmap, then move to GPU + self._load_module_weights_from_mmap(m, dit_name, name) + # Materialize any remaining meta buffers (e.g. RoPE + # inv_freq, timestep freqs) on device and + # recompute them from their original formulas. + # These are non-persistent buffers not present in + # the checkpoint — they must be reconstructed, not + # left as torch.empty. + # NOTE: Do NOT call reset_parameters() here — it + # would re-initialize ALL parameters (including + # mmap-loaded weights) with random values. + _has_meta_buf = any(hasattr(b, "is_meta") and b.is_meta for b in m.buffers(recurse=True)) + if _has_meta_buf: + # Save mmap-loaded param values before + # reset_parameters (which re-initializes + # ALL params with random values). + saved_params = { + n: p.data.clone() + for n, p in m.named_parameters() + if not (hasattr(p, "is_meta") and p.is_meta) + } + m.to_empty(device=self.device) + for submod in m.modules(): + if hasattr(submod, "reset_parameters"): + submod.reset_parameters() + # Restore mmap-loaded weights + for n, p in m.named_parameters(): + if n in saved_params: + p.data.copy_(saved_params[n]) + try: + m.to(self.device) + except Exception: + logger.warning("Module %s still has meta params after mmap load", name) + else: + logger.debug(f"Skipped blocks module {name}") + + # Move top-level params/buffers to GPU + for param in dit_module._parameters.values(): + if param is not None and not (hasattr(param, "is_meta") and param.is_meta): + param.data = param.data.to(self.device, non_blocking=True) + for buffer in dit_module._buffers.values(): + if buffer is not None: + buffer.data = buffer.data.to(self.device, non_blocking=True) + + # Register hooks in a circular sliding window: + # last block prefetches first block, block i prefetches block (i+1) + # All hooks share 2 global device buffers (RFC: "exactly two layers on device") + last_block, first_block = blocks[-1], blocks[0] + + # Pass 1: create hooks with deferred buffer allocation + # (shared_buffers=[None,None] prevents per-hook OOM on large models) + last_hook = apply_distributed_block_hook( + last_block, + first_block, + self.device, + self.dp_group, + self.dp_size, + self.rank, + self.copy_stream, + self.comm_stream, + self.config.pin_cpu_memory, + shared_buffers=[None, None], + ) + + block_hooks: list[DistributedLayerwiseOffloadHook] = [last_hook] + for i, block in enumerate(blocks[:-1]): + next_block = blocks[(i + 1) % num_blocks] + hook = apply_distributed_block_hook( + block, + next_block, + self.device, + self.dp_group, + self.dp_size, + self.rank, + self.copy_stream, + self.comm_stream, + self.config.pin_cpu_memory, + shared_buffers=[None, None], + ) + block_hooks.append(hook) + + # Wire backward references for cache-dit fallback + for i in range(len(block_hooks)): + block_hooks[i]._prev_hook = block_hooks[i - 1] + + # Assign slots in list order: block_hooks = [last_hook, block0, ..., blockN-2] + # This ensures last_hook.current_slot != block0_hook.current_slot, + # so the circular prefetch (last_hook -> block0) writes to a + # different slot than block0 reads from. Correct for ALL N. + for i, hook in enumerate(block_hooks): + hook.current_slot = i % 2 + + # Mark block0_hook (index 1) as group-first + if len(block_hooks) > 1: + block_hooks[1]._is_group_first = True + + # Defer buffer allocation — collected for unified allocation below + self._all_hook_groups.append(block_hooks) + self._blocks.append(blocks) + + if not self._all_hook_groups: + return + + # Unified allocation: 2 shared output buffers + 2 shared shard buffers + # sized to the max block across ALL module groups (gen_layers + + # language_model). Groups execute sequentially, so 2 buffers suffice. + all_hooks: list[DistributedLayerwiseOffloadHook] = [] + for group in self._all_hook_groups: + all_hooks.extend(group) + + unified_buffers = self._allocate_shared_buffers(all_hooks) + unified_shard_buffers = None + if self.dp_size > 1: + unified_shard_buffers = self._allocate_shared_shard_buffers(all_hooks) + + # Shared slot-group tracker: _shared_slot_group[slot] = group_id + # that last wrote to that slot. Group-first hooks use this to + # skip sync-prefetch when the slot still contains their own data. + shared_slot_group = [-1, -1] + + for group_idx, group in enumerate(self._all_hook_groups): + for hook in group: + hook.gpu_buffers = unified_buffers + hook._owns_buffers = False + if unified_shard_buffers is not None: + hook.gpu_shard_buffers = unified_shard_buffers + hook._group_id = group_idx + hook._shared_slot_group = shared_slot_group + + # Prefetch first block of the FIRST module group only. + # Subsequent groups share the same 2 device buffers; prefetching + # them now would overwrite the first group's data in the shared + # buffer (both groups default to slot 0). Instead, subsequent + # groups' first blocks remain as meta placeholders, and their + # pre_forward will sync-prefetch on-demand via the is_materialized + # check — by which point the first group's forward has completed + # and its buffer slots are free. + if self._all_hook_groups: + group = self._all_hook_groups[0] + first_slot = group[0].current_slot + group[-1].prefetch_layer(slot=first_slot, non_blocking=False) + group[-1].get_weights(first_slot) + + total_blocks = sum(len(b) for b in self._blocks) + logger.info( + f"Distributed layer-wise offloading enabled on {total_blocks} blocks " + f"across {len(self._all_hook_groups)} group(s), dp_size={self.dp_size}, " + f"unified shared_buffers=2" + ) + + self.enabled = True + + # Release mmap file handles — _shard_and_pin has copied all shards, + # params now point to offload placeholders (not mmap views). + self._release_mmap_handles() + + # Assign GPU buffers to sharded on-demand modules (VAE/encoders). + # Each module gets a dedicated input (shard-sized) and output + # (full-sized) buffer. VAE/encoders run before/after DiT, never + # concurrently, so peak HBM = max(DiT, VAE) not sum. + for si in self._on_demand_shard_infos: + out_bufs: dict[torch.dtype, torch.Tensor] = {} + in_bufs: dict[torch.dtype, torch.Tensor] = {} + for dtype, shard in si["cpu_shards"].items(): + # AllGather output = dp_size * shard_size + full_size = shard.numel() * self.dp_size if self.dp_size > 1 else shard.numel() + out_bufs[dtype] = torch.empty(full_size, dtype=dtype, device=self.device) + in_bufs[dtype] = torch.empty(shard.shape, dtype=dtype, device=self.device) + si["gpu_output"] = out_bufs + si["gpu_input"] = in_bufs + _mb = sum(t.nelement() * t.element_size() for t in out_bufs.values()) / 1048576 + logger.info("Allocated %.0f MB GPU buffer for sharded on-demand module", _mb) + + self._cleanup_after_loading() + + def _release_mmap_handles(self) -> None: + """Release safetensors mmap file handles.""" + if hasattr(self, "_mmap_file_cache"): + self._mmap_file_cache.clear() + del self._mmap_file_cache + if hasattr(self, "_mmap_model_to_ckpt"): + del self._mmap_model_to_ckpt + logger.info("Released safetensors mmap file handles") + + def _cleanup_after_loading(self) -> None: + """Synchronize and release freed device/CPU memory after sharding.""" + current_omni_platform.synchronize() + current_omni_platform.empty_cache() + import ctypes as _ctypes + import gc as _gc + + _gc.collect() + try: + _ctypes.CDLL("libc.so.6").malloc_trim(0) + except Exception: + pass + + def disable(self) -> None: + if not self.enabled: + return + + for blocks in self._blocks: + for block in blocks: + remove_distributed_block_hook(block) + + for h in getattr(self, "_on_demand_handles", []): + h.remove() + self._on_demand_handles = [] + self._on_demand_shard_infos = [] + + self._blocks.clear() + self._all_hook_groups.clear() + self.enabled = False + logger.info("Distributed layer-wise offloading disabled") + + @staticmethod + def _allocate_shared_buffers( + hooks: list[DistributedLayerwiseOffloadHook], + ) -> list[dict[torch.dtype, torch.Tensor] | None]: + """Allocate exactly 2 shared device buffers sized to the largest block. + + All hooks share these 2 buffers. At any time, slot 0 holds the current + layer's weights and slot 1 holds the next layer's weights (or vice + versa). This ensures only 2 layers' worth of weights reside on device, + regardless of the total number of blocks. + """ + max_sizes: dict[torch.dtype, int] = {} + for hook in hooks: + dp = hook.dp_size + for dtype, metas in hook.metadata.items(): + total = sum(m["numel"] for m in metas) + # AllGather output = dp * ceil(total/dp) (padded for equal shards) + if dp > 1: + total = ((total + dp - 1) // dp) * dp + if dtype not in max_sizes or total > max_sizes[dtype]: + max_sizes[dtype] = total + + device = hooks[0].device + shared_buffers: list[dict[torch.dtype, torch.Tensor] | None] = [None, None] + for slot in range(2): + gpu_weights: dict[torch.dtype, torch.Tensor] = {} + for dtype, total_numel in max_sizes.items(): + gpu_weights[dtype] = torch.empty(total_numel, dtype=dtype, device=device) + shared_buffers[slot] = gpu_weights + + logger.info( + "Allocated 2 shared device buffers (max block size: %s)", + {str(k): f"{v * _dtype_size(k) / 1024 / 1024:.1f}MB" for k, v in max_sizes.items()}, + ) + return shared_buffers + + @staticmethod + def _allocate_shared_shard_buffers( + hooks: list[DistributedLayerwiseOffloadHook], + ) -> list[dict[torch.dtype, torch.Tensor] | None]: + """Allocate 2 shared shard (AllGather input) buffers sized to the + largest per-rank shard. + + All hooks share these 2 input buffers — the same device address is + reused for every layer's AllGather so HCCL can reuse its internal + communication buffers (preventing rank-dependent HBM imbalance). + Cost: 2 × max_shard_size (~184 MB) instead of 2 × N_hooks × shard. + """ + max_shard_sizes: dict[torch.dtype, int] = {} + for hook in hooks: + for dtype, shard in hook.cpu_shards.items(): + numel = shard.numel() + if dtype not in max_shard_sizes or numel > max_shard_sizes[dtype]: + max_shard_sizes[dtype] = numel + + device = hooks[0].device + shared_shard_buffers: list[dict[torch.dtype, torch.Tensor] | None] = [None, None] + for slot in range(2): + shard_bufs: dict[torch.dtype, torch.Tensor] = {} + for dtype, numel in max_shard_sizes.items(): + shard_bufs[dtype] = torch.empty(numel, dtype=dtype, device=device) + shared_shard_buffers[slot] = shard_bufs + + logger.info( + "Allocated 2 shared shard buffers (max shard size: %s)", + {str(k): f"{v * _dtype_size(k) / 1024 / 1024:.1f}MB" for k, v in max_shard_sizes.items()}, + ) + return shared_shard_buffers diff --git a/vllm_omni/diffusion/offloader/module_collector.py b/vllm_omni/diffusion/offloader/module_collector.py index 17d4ca2fd..d6fcb2569 100644 --- a/vllm_omni/diffusion/offloader/module_collector.py +++ b/vllm_omni/diffusion/offloader/module_collector.py @@ -80,11 +80,13 @@ class ModuleDiscovery: "text_encoder_2", "text_encoder_3", "image_encoder", + "vision_encoder", "mllm", ] _FALLBACK_VAE_ATTRS = [ "vae", "audio_vae", + "sound_tokenizer", ] @staticmethod diff --git a/vllm_omni/diffusion/offloader/offload_plan.py b/vllm_omni/diffusion/offloader/offload_plan.py new file mode 100644 index 000000000..575194ea0 --- /dev/null +++ b/vllm_omni/diffusion/offloader/offload_plan.py @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Declarative OffloadPlan for distributed layerwise offload. + +Models declare this as a class attribute ``_offload_plan`` on the +pipeline class. When present, the offloader uses it instead of +heuristic block discovery. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from torch import nn + + +@dataclass(frozen=True) +class OffloadPlan: + """Optional declarative metadata for distributed layerwise offload. + + Models declare this as a class attribute ``_offload_plan`` on the + pipeline class. When present, the offloader uses it instead of + heuristic block discovery, making it easier to support new models + without offload-specific logic in constructor or forward code. + + If not declared, the offloader falls back to: + 1. ``_layerwise_offload_blocks_attrs`` on each DiT module class. + 2. Heuristic search for ``layers`` / ``blocks`` / ``h`` attributes. + + Attributes: + block_attrs: Maps DiT path → tuple of block-list attribute names. + e.g. ``{"transformer": ("gen_layers",), + "transformer.language_model": ("layers",)}`` + offload_submodules: Maps child name → block-list attribute name, + for large non-DiT submodules within a DiT that should be + independently offloaded with their own hooks. + e.g. ``{"context_encoder": "layers"}`` + """ + + block_attrs: dict[str, tuple[str, ...]] = field(default_factory=dict) + offload_submodules: dict[str, str] = field(default_factory=dict) + + +def get_offload_plan(pipeline: nn.Module) -> OffloadPlan | None: + """Retrieve the OffloadPlan declared by the pipeline, if any.""" + return getattr(pipeline, "_offload_plan", None) + + +def supports_mmap_loading(pipeline: nn.Module) -> bool: + """Whether the pipeline supports mmap weight loading. + + A pipeline supports mmap if any module defines ``_remap_ckpt_key``, + which maps checkpoint keys to model parameter names. Models without + this method must use the regular ``load_weights()`` path. + + This check is shared between ``diffusers_loader.py`` (to decide whether + to skip ``load_weights``) and ``DistributedLayerwiseOffloadBackend.enable()`` + (to decide whether to call ``_load_weights_via_mmap``), ensuring both + paths use the same gate condition. + """ + return any(callable(getattr(type(m), "_remap_ckpt_key", None)) for m in pipeline.modules()) diff --git a/vllm_omni/diffusion/offloader/tensor_utils.py b/vllm_omni/diffusion/offloader/tensor_utils.py new file mode 100644 index 000000000..ef2d96011 --- /dev/null +++ b/vllm_omni/diffusion/offloader/tensor_utils.py @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Shared tensor utilities for distributed layerwise offload. + +These helpers are used by both DistributedLayerwiseOffloadHook and +DistributedLayerwiseOffloadBackend, and can be reused by other +offload backends. +""" + +from __future__ import annotations + +import torch +from torch.distributed.tensor import DTensor + + +def dtype_size(dtype: torch.dtype) -> int: + """Return element size in bytes for a torch.dtype.""" + return torch.empty(1, dtype=dtype).element_size() + + +def is_dtensor(t: torch.Tensor) -> bool: + """Check if tensor is a DTensor.""" + return isinstance(t, DTensor) + + +def set_tensor_storage(target: torch.Tensor, value: torch.Tensor) -> None: + """Replace target's underlying storage with value (zero-copy).""" + if is_dtensor(target): + target._local_tensor = value + else: + target.data = value + + +def make_offload_placeholder(tensor: torch.Tensor) -> torch.Tensor: + """Create a zero-element placeholder to free GPU memory.""" + if is_dtensor(tensor): + local_shape = tuple(tensor.to_local().shape) + return torch.empty(local_shape, device="meta", dtype=tensor.dtype) + return torch.empty((0,), device=tensor.device, dtype=tensor.dtype) + + +def is_materialized_tensor(t: torch.Tensor) -> bool: + """Check if tensor holds real data (not meta or empty placeholder).""" + if is_dtensor(t): + local_t = t.to_local() + return not local_t.is_meta + return not t.is_meta and t.data.numel() > 0 diff --git a/vllm_omni/diffusion/worker/diffusion_model_runner.py b/vllm_omni/diffusion/worker/diffusion_model_runner.py index be68f071f..3255e59db 100644 --- a/vllm_omni/diffusion/worker/diffusion_model_runner.py +++ b/vllm_omni/diffusion/worker/diffusion_model_runner.py @@ -212,7 +212,11 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): ) load_device = ( - "cpu" if self.od_config.enable_cpu_offload or self.od_config.enable_layerwise_offload else str(self.device) + "cpu" + if self.od_config.enable_cpu_offload + or self.od_config.enable_layerwise_offload + or getattr(self.od_config, "enable_distributed_layerwise_offload", False) + else str(self.device) ) def get_memory_context() -> AbstractContextManager[Any]: @@ -478,7 +482,8 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin): # better perf. HSDP2's fully_shard pre-forward hooks need tensor version # counters, which inference tensors do not track. use_hsdp = od_config.parallel_config.use_hsdp - grad_context = torch.no_grad() if use_hsdp else torch.inference_mode() + use_distributed_offload = getattr(self.od_config, "enable_distributed_layerwise_offload", False) + grad_context = torch.no_grad() if (use_hsdp or use_distributed_offload) else torch.inference_mode() with grad_context: for req in reqs: self._prepare_request_for_forward( diff --git a/vllm_omni/diffusion/worker/diffusion_worker.py b/vllm_omni/diffusion/worker/diffusion_worker.py index 58c8fd1eb..9a2958fcf 100644 --- a/vllm_omni/diffusion/worker/diffusion_worker.py +++ b/vllm_omni/diffusion/worker/diffusion_worker.py @@ -417,12 +417,34 @@ class DiffusionWorker: def execute_model( self, - req: OmniDiffusionRequest, + req: OmniDiffusionRequest | list[OmniDiffusionRequest], od_config: OmniDiffusionConfig, kv_prefetch_job: KVPrefetchJob | None = None, ) -> DiffusionOutput: - """Execute a forward pass by delegating to the model runner.""" + """Execute a forward pass by delegating to the model runner. + + If *req* is a list (DP multi-concurrency), each rank picks one + request based on its distributed rank. AllGather in the layerwise + offload only gathers weight shards (request-independent), so all + ranks stay synchronised at each AllGather call while computing + different activations. + + Each rank returns its OWN DiffusionOutput (no gather). The executor + collects N responses via result_mq (all ranks share one queue). + """ assert self.model_runner is not None, "Model runner not initialized" + + # DP multi-concurrency: pick one request per DP rank. + # Use rank_in_group (DP rank) not global rank, so that SP/TP/CFG + # ranks within the same DP replica select the same request. + is_batch = isinstance(req, list) + if is_batch: + from vllm_omni.diffusion.distributed.parallel_state import get_data_parallel_rank + + dp_rank = get_data_parallel_rank() + idx = dp_rank % len(req) + req = req[idx] + if self.lora_manager is not None: try: self.lora_manager.set_active_adapter(req.sampling_params.lora_request, req.sampling_params.lora_scale) @@ -436,6 +458,14 @@ class DiffusionWorker: output = self.model_runner.execute_model(req, kv_prefetch_job=kv_prefetch_job) if profiler: profiler.step() + + # Each primary rank returns its own output tagged with its DP rank + # so the executor can match results to requests by dp_rank. + # Non-primary ranks (SP/TP > 0) do not reply. + if is_batch: + from vllm_omni.diffusion.distributed.parallel_state import get_data_parallel_rank + + return {"dp_rank": get_data_parallel_rank(), "output": output} return output def execute_model_batch( @@ -781,17 +811,12 @@ class WorkerProc: self.result_mq = None self.result_mq_handle = None - # Setup result sender (only for rank 0) - if gpu_id == 0: - self.result_mq = MessageQueue(n_reader=1, n_local_reader=1, local_reader_ranks=[0]) - self.result_mq_handle = self.result_mq.export_handle() - WorkerProc._shared_result_handle = self.result_mq_handle - logger.info(f"Worker {gpu_id} created result MessageQueue") - else: - handle = getattr(WorkerProc, "_shared_result_handle", None) - if handle: - self.result_mq = MessageQueue.create_from_handle(handle, gpu_id) - logger.info(f"Worker {gpu_id} attached to shared result MessageQueue") + # Each worker creates its own result MessageQueue (as writer). + # The executor creates one reader per worker to collect responses. + # This supports DP multi-concurrency where all ranks reply independently. + self.result_mq = MessageQueue(n_reader=1, n_local_reader=1, local_reader_ranks=[0]) + self.result_mq_handle = self.result_mq.export_handle() + logger.info(f"Worker {gpu_id} created result MessageQueue") assert od_config.master_port is not None @@ -920,12 +945,33 @@ class WorkerProc: output_rank = rpc_request.get("output_rank") exec_all_ranks = rpc_request.get("exec_all_ranks", False) collect_rank_status = rpc_request.get("collect_rank_status", False) + wave_id = rpc_request.get("wave_id") if collect_rank_status and not exec_all_ranks: raise ValueError("collect_rank_status requires exec_all_ranks=True so all ranks enter the status gather") should_execute = exec_all_ranks or output_rank is None or output_rank == self.gpu_id - should_reply = (output_rank is None or output_rank == self.gpu_id) and self.result_mq is not None + # For DP multi-concurrency (output_rank=None), only the primary rank + # within each DP replica should reply. This prevents SP/TP/CFG/PP + # ranks from enqueuing extra replies that the executor doesn't drain. + if output_rank is None and exec_all_ranks: + from vllm.distributed.parallel_state import get_tensor_model_parallel_rank + + from vllm_omni.diffusion.distributed.parallel_state import ( + get_classifier_free_guidance_rank, + get_pipeline_parallel_rank, + get_sequence_parallel_rank, + ) + + is_primary_in_replica = ( + get_sequence_parallel_rank() == 0 + and get_classifier_free_guidance_rank() == 0 + and get_tensor_model_parallel_rank() == 0 + and get_pipeline_parallel_rank() == 0 + ) + should_reply = is_primary_in_replica and self.result_mq is not None + else: + should_reply = (output_rank is None or output_rank == self.gpu_id) and self.result_mq is not None if not should_execute: return None, False @@ -968,6 +1014,7 @@ class WorkerProc: "method": method, "result": result, "rank_statuses": rank_statuses, + "wave_id": wave_id, }, True, ) @@ -975,6 +1022,8 @@ class WorkerProc: if rpc_exception is not None: raise rpc_exception + if isinstance(result, dict) and wave_id is not None: + result["wave_id"] = wave_id return result, should_reply def _worker_busy_loop(self) -> None: @@ -1016,10 +1065,16 @@ class WorkerProc: self._return_result(result, rpc_id=rpc_id) except Exception as e: logger.error(f"Error processing RPC: {e}", exc_info=True) + # Apply the same reply gate as the success path so + # non-output ranks don't enqueue stale error replies + # that compete with the expected responder's message. + output_rank = msg.get("output_rank") + exec_all_ranks = msg.get("exec_all_ranks", False) + wave_id = msg.get("wave_id") if self.result_mq is not None: if rpc_id is not None: - # Async RPC: must complete the executor''s pending - # future so collective_rpc() doesn''t hang. + # Async RPC: must complete the executor's pending + # future so collective_rpc() doesn't hang. self.result_mq.enqueue( AsyncDiffusionOutput( kind=AsyncOutputKind.RPC_RESULT, @@ -1027,8 +1082,36 @@ class WorkerProc: error=str(e), ) ) - else: - self._return_result({"status": "error", "error": str(e)}) + elif output_rank is None and exec_all_ranks: + # DP multi-concurrency: primary ranks reply, tagged + from vllm.distributed.parallel_state import ( + get_tensor_model_parallel_rank, + ) + + from vllm_omni.diffusion.distributed.parallel_state import ( + get_classifier_free_guidance_rank, + get_data_parallel_rank, + get_pipeline_parallel_rank, + get_sequence_parallel_rank, + ) + + is_primary = ( + get_sequence_parallel_rank() == 0 + and get_classifier_free_guidance_rank() == 0 + and get_tensor_model_parallel_rank() == 0 + and get_pipeline_parallel_rank() == 0 + ) + if is_primary: + try: + dp_rank = get_data_parallel_rank() + except Exception: + dp_rank = self.gpu_id + self._return_result( + {"status": "error", "error": str(e), "dp_rank": dp_rank, "wave_id": wave_id} + ) + elif output_rank is None or output_rank == self.gpu_id: + # Normal RPC: only the expected rank replies + self._return_result({"status": "error", "error": str(e), "wave_id": wave_id}) elif isinstance(msg, dict) and msg.get("type") == "shutdown": logger.info("Worker %s: Received shutdown message", self.gpu_id) @@ -1103,7 +1186,7 @@ class WorkerProc: pipe_writer.send( { "status": "ready", - "result_handle": worker_proc.result_mq_handle if rank == 0 else None, + "result_handle": worker_proc.result_mq_handle, } ) worker_proc._worker_busy_loop() diff --git a/vllm_omni/engine/arg_utils.py b/vllm_omni/engine/arg_utils.py index ed3f5464d..096649b1f 100644 --- a/vllm_omni/engine/arg_utils.py +++ b/vllm_omni/engine/arg_utils.py @@ -480,6 +480,8 @@ class OrchestratorArgs: num_weight_load_threads: int = 4 enable_cpu_offload: bool = False enable_layerwise_offload: bool = False + enable_distributed_layerwise_offload: bool = False + dlo_use_allgather: bool = True boundary_ratio: float | None = None flow_shift: float | None = None diffusion_kv_cache_dtype: str | None = None diff --git a/vllm_omni/engine/async_omni_engine.py b/vllm_omni/engine/async_omni_engine.py index a01d905b1..1fd747df5 100644 --- a/vllm_omni/engine/async_omni_engine.py +++ b/vllm_omni/engine/async_omni_engine.py @@ -1008,6 +1008,8 @@ class AsyncOmniEngine: "enable_cache_dit_summary": kwargs.get("enable_cache_dit_summary", False), "enable_cpu_offload": kwargs.get("enable_cpu_offload", False), "enable_layerwise_offload": kwargs.get("enable_layerwise_offload", False), + "enable_distributed_layerwise_offload": kwargs.get("enable_distributed_layerwise_offload", False), + "dlo_use_allgather": kwargs.get("dlo_use_allgather", True), "enforce_eager": False if kwargs.get("enforce_eager") is None else kwargs.get("enforce_eager"), "diffusion_compile_granularity": ( "regional" diff --git a/vllm_omni/entrypoints/cli/serve.py b/vllm_omni/entrypoints/cli/serve.py index 83e975528..2d014891e 100644 --- a/vllm_omni/entrypoints/cli/serve.py +++ b/vllm_omni/entrypoints/cli/serve.py @@ -641,6 +641,30 @@ class OmniServeCommand(CLISubcommand): action="store_true", help="Enable layerwise (blockwise) offloading on DiT modules.", ) + omni_config_group.add_argument( + "--enable-distributed-layerwise-offload", + action="store_true", + help="Enable distributed layerwise offloading with H2D + AllGather overlap. " + "Shards weights across DP ranks, stores only 1/DP_size on each host, " + "and overlaps H2D transfers and AllGather with computation. " + "DP size is automatically derived from the parallel configuration.", + ) + omni_config_group.add_argument( + "--dlo-use-allgather", + dest="dlo_use_allgather", + action="store_true", + default=True, + help="Use shard + AllGather for weight reconstruction (default: True). " + "When disabled (--dlo-no-use-allgather), each rank loads full weights " + "via H2D only — no sharding, no AllGather, no concurrent request " + "requirement, but N× CPU memory.", + ) + omni_config_group.add_argument( + "--dlo-no-use-allgather", + dest="dlo_use_allgather", + action="store_false", + help="Disable AllGather: each rank loads full weights independently.", + ) # Video model parameters (e.g., Wan2.2) - engine-level omni_config_group.add_argument( "--boundary-ratio",